def query(statement):
cluster = Cluster(['beehive-cassandra'])
session = cluster.connect('waggle')
rows = session.execute(statement)
return cluster, rows
def benchmark(thread_class):
options, args = parse_options()
for conn_class in options.supported_reactors:
setup(options)
log.info("==== %s ====" % (conn_class.__name__, ))
kwargs = {
'metrics_enabled': options.enable_metrics,
'connection_class': conn_class
}
if options.protocol_version:
kwargs['protocol_version'] = options.protocol_version
cluster = Cluster(options.hosts, **kwargs)
session = cluster.connect(options.keyspace)
log.debug("Sleeping for two seconds...")
time.sleep(2.0)
# Generate the query
if options.read:
query = "SELECT * FROM {0} WHERE thekey = '{{key}}'".format(TABLE)
else:
query = "INSERT INTO {0} (thekey".format(TABLE)
for i in range(options.num_columns):
query += ", col{0}".format(i)
query += ") VALUES ('{key}'"
for i in range(options.num_columns):
query += ", {0}".format(COLUMN_VALUES[options.column_type])
query += ")"
values = None # we don't use that anymore. Keeping it in case we go back to prepared statements.
per_thread = options.num_ops // options.threads
threads = []
log.debug(
"Beginning {0}...".format('reads' if options.read else 'inserts'))
start = time.time()
try:
for i in range(options.threads):
thread = thread_class(i, session, query, values, per_thread,
cluster.protocol_version,
options.profile)
thread.daemon = True
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
while thread.is_alive():
thread.join(timeout=0.5)
end = time.time()
finally:
cluster.shutdown()
teardown(options)
total = end - start
log.info("Total time: %0.2fs" % total)
log.info("Average throughput: %0.2f/sec" % (options.num_ops / total))
if options.enable_metrics:
stats = scales.getStats()['cassandra']
log.info("Connection errors: %d", stats['connection_errors'])
log.info("Write timeouts: %d", stats['write_timeouts'])
log.info("Read timeouts: %d", stats['read_timeouts'])
log.info("Unavailables: %d", stats['unavailables'])
log.info("Other errors: %d", stats['other_errors'])
log.info("Retries: %d", stats['retries'])
request_timer = stats['request_timer']
log.info("Request latencies:")
log.info(" min: %0.4fs", request_timer['min'])
log.info(" max: %0.4fs", request_timer['max'])
log.info(" mean: %0.4fs", request_timer['mean'])
log.info(" stddev: %0.4fs", request_timer['stddev'])
log.info(" median: %0.4fs", request_timer['median'])
log.info(" 75th: %0.4fs", request_timer['75percentile'])
log.info(" 95th: %0.4fs", request_timer['95percentile'])
log.info(" 98th: %0.4fs", request_timer['98percentile'])
log.info(" 99th: %0.4fs", request_timer['99percentile'])
log.info(" 99.9th: %0.4fs", request_timer['999percentile'])
# bool flag database connections
cassandraDbOn=False
neo4jDbOn=False
def dbCleanUP(cassandraDbOn,neo4jDbOn):
if cassandraDbOn:
print 'cleaning Cassandra state'
session.shutdown();
cluster.shutdown();
atexit.register(dbCleanUP,True,True)
# setup cassandra connection
cassandraDbOn=True
cluster = Cluster()
session = cluster.connect('learner_db')
# set response schema to Dictionaries
session.row_factory = dict_factory
def moveConceptMap():
# neo4j graph connector
graph = Graph()
# load concept map from production
import requests
url = "https://api.ekstep.in/learning/v2/domains/numeracy/concepts"
from cassandra.cluster import Cluster
import sys
import logging
import traceback
import uuid
import datetime
import time
cluster = Cluster(['iaccounts-cassandra-1'])
session = cluster.connect('organization')
try:
print "***************************************"
print "Check if the organization exists with the TIN provided...."
buffer = 'select org_id from org where org_name=' + '\'' + sys.argv[
1] + '\'' + ' and org_tin=' + '\'' + sys.argv[2] + '\''
print buffer
rows = session.execute(buffer)
print rows
if rows:
print rows[0].org_id
uuidbuff = str(rows[0].org_id)
newuuid = ''
for i in uuidbuff:
if i == '-':
newuuid = newuuid + '_'
else:
newuuid = newuuid + i
print newuuid
else:
print "***************************************"
print "Failed to fetch the organization....Terminating...."
# upload mnist objects to cassandra cluster
from cassandra.cluster import Cluster
import os
from timeit import default_timer as timer
import pickle
from cassandra.encoder import Encoder
import math
import numpy as np
import time
from cassandra import query
cluster = Cluster(['10.10.1.1', '10.10.1.1'])
session = cluster.connect('testkeyspace')
start = timer()
data = {}
pre = session.prepare("SELECT * FROM mnist_dictstr WHERE id IN ?")
dataset_size = 60000
idx = [i for i in range(0, dataset_size)]
batch_size = 32
batch_num = int(math.ceil(dataset_size / batch_size))
for epoch in range(1, 2):
np.random.seed(9)
np.random.shuffle(idx)
print(idx[0:5])
start = time.time()
for i in range(0, batch_num):
idx_batch = idx[batch_size * i:min(dataset_size, batch_size * (i + 1))]
def test_dc_aware_roundrobin_one_remote_host(self):
use_multidc([2, 2])
keyspace = 'test_dc_aware_roundrobin_one_remote_host'
cluster = Cluster(
load_balancing_policy=DCAwareRoundRobinPolicy('dc2', used_hosts_per_remote_dc=1),
protocol_version=PROTOCOL_VERSION)
session = cluster.connect()
wait_for_up(cluster, 1, wait=False)
wait_for_up(cluster, 2, wait=False)
wait_for_up(cluster, 3, wait=False)
wait_for_up(cluster, 4)
create_schema(session, keyspace, replication_strategy=[2, 2])
self._insert(session, keyspace)
self._query(session, keyspace)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 0)
self.coordinator_stats.assert_query_count_equals(self, 3, 6)
self.coordinator_stats.assert_query_count_equals(self, 4, 6)
self.coordinator_stats.reset_counts()
bootstrap(5, 'dc1')
wait_for_up(cluster, 5)
self._query(session, keyspace)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 0)
self.coordinator_stats.assert_query_count_equals(self, 3, 6)
self.coordinator_stats.assert_query_count_equals(self, 4, 6)
self.coordinator_stats.assert_query_count_equals(self, 5, 0)
self.coordinator_stats.reset_counts()
decommission(3)
decommission(4)
wait_for_down(cluster, 3, wait=True)
wait_for_down(cluster, 4, wait=True)
self._query(session, keyspace)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
self.coordinator_stats.assert_query_count_equals(self, 4, 0)
responses = set()
for node in [1, 2, 5]:
responses.add(self.coordinator_stats.get_query_count(node))
self.assertEqual(set([0, 0, 12]), responses)
self.coordinator_stats.reset_counts()
decommission(5)
wait_for_down(cluster, 5, wait=True)
self._query(session, keyspace)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
self.coordinator_stats.assert_query_count_equals(self, 4, 0)
self.coordinator_stats.assert_query_count_equals(self, 5, 0)
responses = set()
for node in [1, 2]:
responses.add(self.coordinator_stats.get_query_count(node))
self.assertEqual(set([0, 12]), responses)
self.coordinator_stats.reset_counts()
decommission(1)
wait_for_down(cluster, 1, wait=True)
self._query(session, keyspace)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 12)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
self.coordinator_stats.assert_query_count_equals(self, 4, 0)
self.coordinator_stats.assert_query_count_equals(self, 5, 0)
self.coordinator_stats.reset_counts()
force_stop(2)
try:
self._query(session, keyspace)
self.fail()
except NoHostAvailable:
pass
cluster.shutdown()
def insertData(file_name,number):
cluster = Cluster(contact_points=['172.18.0.3'],port=9042)
session = cluster.connect()
session.execute("use %s" % KEYSPACE)
session.execute('INSERT INTO USerInfo(timestamp,imagename,predict_number)VALUES(toTimestamp(now()),%s,%s);',[file_name,number])
userIDtable = dbSession.execute(sql)
if userIDtable is not None:
print "%s=(%d\t%d)" % (userID, userPasswd, userIDtable[0].passwd)
if (userPasswd > userIDtable[0].passwd):
guessPasswd(dbSession, userID, fromNum, userIDtable[0].passwd)
elif (userPasswd < userIDtable[0].passwd):
guessPasswd(dbSession, userID, userIDtable[0].passwd, toNum)
else:
print "%s=%d" % (userID, userPasswd)
############################################################################
startTime = time.clock()
cluster = Cluster()
session = cluster.connect('emprogria')
dbCluster = Cluster()
dbSession = dbCluster.connect('emprogria')
result = dbSession.execute('TRUNCATE fuhu;')
recNum = 100
if len(sys.argv) > 1:
recNum = int(sys.argv[1])
############################################################################
userIDs = []
insertBatch(dbSession, recNum)
writer.writerow(data)
def load_geo_data(session, geopath):
geo_prepared = session.prepare(GEO_INSERT_STATEMENT)
with open(geopath, 'rb') as f:
reader = csv.reader(f)
for row in reader:
geo_bound = geo_prepared.bind(map(uuid.UUID, row[0:2]) + row[2:])
session.execute(geo_bound)
if __name__ == '__main__':
meta_path = 'metadata.json'
geo_path = 'geodata.csv'
cluster = Cluster(['node0', 'node1', 'node2'])
session = cluster.connect()
session.execute(META_CF_DROP_STATEMENT)
session.execute(RANK_CF_DROP_STATEMENT)
session.execute(GEO_CF_DROP_STATEMENT)
session.execute(KS_CREATION_STATEMENT)
session.execute(GEO_CF_CREATION_STATEMENT)
session.execute(META_CF_CREATION_STATEMENT)
session.execute(RANK_CF_CREATION_STATEMENT)
meta_prepared = session.prepare(META_INSERT_STATEMENT)
rank_prepared = session.prepare(RANK_INSERT_STATEMENT)
print('loading geo')
load_geo_data(session, geo_path)
print('loading meta')
for data in parse(meta_path):
'''#############################################################################
The number of entries to add in each table.
#############################################################################'''
num_articles = 10
num_comments = 10
num_tags = 10
num_users = 10
new_user_ids = []
new_article_ids = []
'''#############################################################################
Setup Cassandra, connect to a cluster and keyspace.
#############################################################################'''
from cassandra.cluster import Cluster
cluster = Cluster(['172.17.0.2'])
session = cluster.connect()
session.set_keyspace('beepboopbackend')
'''#############################################################################
Fill the users table
#############################################################################'''
for x in range(0, num_users):
newid = uuid.uuid1()
new_user_ids.append(newid)
password = "******" + str(x)
pass_hash = hashlib.md5(password.encode())
session.execute(
"""
INSERT INTO users (
user_id,
email,
#returned value = ( "subreddit::Ngram", count)
return ("{0}::{1}".format(splitline[2],splitline[0]), tupdata[1])
def combineData(tokenizer, tupdata, ngram):
# body = tupdata[0]
# utctime = tupdata[1]
# subreddit = tupdata[2]
tokens = tokenizer.Ngrams(tupdata.encode('utf-8'), ngram)
#rtnval = ["{0}".format(y, subreddit) for y in tokens]
return tokens
def pushToCassandraTable1(ngramcount, rdditer, async = True):
tbname = table1
# this needs to be here for distribution to workers
from cassandra.cluster import Cluster
CassandraCluster = Cluster(nodes)
success = False
#try to reconnect if connection is down
while not success:
try:
session = CassandraCluster.connect(keyspace)
session.default_timeout = 60
success = True
except:
success = False
time.sleep(CassandraWait)
#session.default_consistency_level = cassandra.ConsistencyLevel.ALL
#self.session.encoder.mapping[tuple] = self.session.encoder.cql_encode_set_collection
import cassandra
from cassandra.cluster import Cluster
try:
cluster = Cluster([
'127.0.0.1'
]) # If you have a locally installed Apache Cassandra instance
session = cluster.connect()
except Exception as e:
print(e)
try:
session.execute("""
CREATE KEYSPACE IF NOT EXISTS udacity
WITH REPLICATION =
{ 'class' : 'SimpleStrategy', 'replication_factor' : 1 }""")
except Exception as e:
print(e)
try:
session.set_keyspace('udacity')
except Exception as e:
print(e)
query = "CREATE TABLE IF NOT EXISTS music_library "
query = query + "(year int, artist_name text, album_name text, city text, PRIMARY KEY (artist_name))"
try:
session.execute(query)
except Exception as e:
print(e)
import re
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
from cassandra.query import tuple_factory
auth_provider = PlainTextAuthProvider(username='******', password='')
cluster = Cluster(['127.0.0.1'], auth_provider=auth_provider, port=9042)
session = cluster.connect("vendorproject")
session.row_factory = tuple_factory
session.execute("TRUNCATE vendorcontent;")
pattern = []
pattern.append('(?<="projectGuid":")([-0-z]*)(?=")')
pattern.append('(?<="userId":")(\d*)(?=")')
pattern.append('(?<="userFirstName":")([0-z]*)(?=")')
pattern.append('(?<="userLastName":")([0-z]*)(?=")')
pattern.append('(?<="channel":")([-0-z]*)(?=")')
pattern.append('(?<="platform":")([-0-z]*)(?=")')
pattern.append('(?<="landingPageThumbnailUrl":")([a-zA-Z0-9\-\.\:\/]*)')
pattern.append('(?<="shareUrl":")([a-zA-Z0-9\-\.\:\/]*)')
pattern.append('(?<="makeMyOwnUrl":")([a-zA-Z0-9\-\.\:\/]*)')
pattern.append('(?<="editProjectUrl":")([a-zA-Z0-9\-\.\:\/]*)')
pattern.append('(?<="previewTitle":")([\(\) a-zA-Z0-9\-\.\:\/]*)')
pattern.append('(?<="previewSubtitle":")([\(\) a-zA-Z0-9\-\.\:\/]*)')
result = session.execute("SELECT id,api_response_content FROM VENDORIDS;")
for i in result:
insert = []
for regex in pattern:
#print(regex)
from prettytable import PrettyTable
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
#Configuration
contactpoints = ['54.159.232.149', '54.209.96.0']
auth_provider = PlainTextAuthProvider(username='******', password='******')
keyspace = "test"
query = "SELECT * FROM test.table1"
print "Connecting to cluster"
startTime = time.time()
cluster = Cluster(contact_points=contactpoints, auth_provider=auth_provider)
session = cluster.connect(keyspace)
print "Seconds to connect to cluster: ", time.time() - startTime
startTime = time.time()
result = session.execute(query)[:]
print "Seconds to run query: ", time.time() - startTime
output = PrettyTable(["beacon", "date", "readtime", "data"])
for i in result:
output.add_row([i[0], i[1], i[2], i[3]])
except Exception as e:
print("Error {} in {}" .format(str(e), str(row['file_path'])))
insrtStmt= session.prepare("INSERT INTO election_process_audit (process_name,file_name,identifier, error_message,exection_datetime,info_message) VALUES (?,?,?,?,?,?)")
session.execute(insrtStmt, ['Google_vision_img_Generator.py',str(['processing_path']),row['queue_id'],str(e),str(datetime.datetime.now()),'--An Error Occured while converting the image-->'+fName])
insrtStmt= sessionInner.prepare("INSERT INTO votelist_individual_page_queue (queue_id,composite_file_refid,file_name,received_date,last_modified_date,status,status_description,file_path,processing_path) VALUES (?,?,?,?,?,?,?,?,?)")
sessionInner.execute_async(insrtStmt, [row['queue_id'],row['composite_file_refid'],str(row['file_path']),str(datetime.datetime.now()),str(datetime.datetime.now()),'ERROR','Error During Processing',str(row['file_path']),str(['processing_path'])])
#insrtStmt= sessionInner.prepare("INSERT INTO googlevision_inbound_queue(queue_id,composite_file_refid,file_name,received_date,last_modified_date,status,status_description,file_path,processing_path) VALUES (?,?,?,?,?,?,?,?,?)")
#sessionInner.execute_async(insrtStmt, [uniqueID,str(composite_file_refid),file_name,str(datetime.datetime.now()),str(datetime.datetime.now()),'Processing Started','Processing Started',file_name,googleInboundPath])
#print("After Loading the status")
from cassandra.cluster import Cluster
cluster = Cluster(['192.168.1.55', '192.168.1.56'])
session = cluster.connect('electionanalysis')
selectStmtCompFiles= session.prepare("SELECT composite_file_refid FROM votelist_individual_page_queue where STATUS='processing started'")
dfCompositeFiles= set((list(session.execute(selectStmtCompFiles))))
print('------------------------------>Started Procesiing')
#print(dfCompositeFiles)
selectStmtStartedProcessing= session.prepare("SELECT * FROM votelist_individual_page_queue where STATUS='processing started'")
selectStmtComplete= session.prepare("SELECT * FROM votelist_individual_page_queue where STATUS='Complete'")
selectStmtError= session.prepare("SELECT * FROM votelist_individual_page_queue where STATUS='ERROR'")
dfProcessingStarted = pd.DataFrame(list(session.execute(selectStmtStartedProcessing)))
dfProcessingComplete = pd.DataFrame(list(session.execute(selectStmtComplete)))
dfError = pd.DataFrame(list(session.execute(selectStmtError)))
#print('------------------------------>Started Procesiing')
#print(dfProcessingStarted)
selectConfig= session.prepare("SELECT * FROM election_anallysis_configurations")
dfConfig=pd.DataFrame(list(session.execute(selectConfig)))
def handle_noargs(self, **options):
cluster = Cluster(['127.0.0.1'])
session = cluster.connect()
rows = session.execute(
"SELECT * FROM system.schema_keyspaces WHERE keyspace_name='users'"
)
if rows:
msg = ' It looks like you already have a users keyspace.\nDo you '
msg += 'want to delete it and recreate it? All current data will '
msg += 'be deleted! (y/n): '
resp = raw_input(msg)
if not resp or resp[0] != 'y':
print "Ok, then we're done here."
return
session.execute("DROP KEYSPACE users")
session.execute("""
CREATE KEYSPACE users
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}
""")
# create tables
session.set_keyspace("users")
session.execute("""
CREATE TABLE users (
username text PRIMARY KEY,
password text
)
""")
session.execute("""
CREATE TABLE friends (
username text,
friend text,
since timestamp,
PRIMARY KEY (username, friend)
)
""")
session.execute("""
CREATE TABLE followers (
username text,
follower text,
since timestamp,
PRIMARY KEY (username, follower)
)
""")
session.execute("""
CREATE TABLE tweets (
tweet_id uuid PRIMARY KEY,
username text,
body text
)
""")
session.execute("""
CREATE TABLE userline (
username text,
time timeuuid,
tweet_id uuid,
PRIMARY KEY (username, time)
) WITH CLUSTERING ORDER BY (time DESC)
""")
session.execute("""
CREATE TABLE timeline (
username text,
time timeuuid,
tweet_id uuid,
PRIMARY KEY (username, time)
) WITH CLUSTERING ORDER BY (time DESC)
""")
rows = session.execute(
"SELECT * FROM system.schema_keyspaces WHERE keyspace_name='bingo'"
)
if rows:
msg = ' It looks like you already have a Bingo keyspace.\nDo you '
msg += 'want to delete it and recreate it? All current data will '
msg += 'be deleted! (y/n): '
resp = raw_input(msg)
if not resp or resp[0] != 'y':
print "Ok, then we're done here."
return
session.execute("DROP KEYSPACE bingo")
session.execute("""
CREATE KEYSPACE bingo
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}
""")
# create tables
session.set_keyspace("bingo")
session.execute("""
CREATE TABLE bingo.tickets (
game_id uuid,
user text,
a1 int,
a2 int,
a3 int,
a4 int,
a5 int,
a6 int,
a7 int,
a8 int,
a9 int,
b1 int,
b2 int,
b3 int,
b4 int,
b5 int,
b6 int,
b7 int,
b8 int,
b9 int,
c1 int,
c2 int,
c3 int,
c4 int,
c5 int,
c6 int,
c7 int,
c8 int,
c9 int,
PRIMARY KEY ((game_id, user)))
""")
session.execute("""
CREATE TABLE bingo.announcement (
game_id uuid,
announcement_time timestamp,
number int,
PRIMARY KEY (game_id, announcement_time)
) WITH CLUSTERING ORDER BY (announcement_time DESC)
""")
session.execute("""
CREATE TABLE bingo.games (
game_id uuid PRIMARY KEY,
announcement_duration int,
bottom_row text,
created_by text,
end_time timestamp,
four_corners text,
full_house text,
is_active boolean,
middle_row text,
no_of_players int,
start_time timestamp,
top_row text)
""")
print 'All done!'
import json
from kafka import KafkaConsumer
from cassandra.cluster import Cluster
from settings import BTC_BLOCK_TOPIC, BOOSTRAP_SERVER, CASSANDRA_HOST, CASSANDRA_KEYSPACE
consumer = KafkaConsumer(BTC_BLOCK_TOPIC,
bootstrap_servers="%s" % (BOOSTRAP_SERVER),
group_id='bitcoin_consumer_2',
auto_offset_reset='earliest')
cluster = Cluster([CASSANDRA_HOST])
session = cluster.connect()
session.execute(
"CREATE KEYSPACE IF NOT EXISTS bitcoin WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } AND durable_writes = true;"
)
session.execute("USE bitcoin;")
session.execute(
"CREATE TABLE IF NOT EXISTS bitcoin.blocks (hash text, confirmations text, strippedsize text, size text, weight text, height text, version text, version_hex text, merkleroot text, time text, mediantime text, nonce text, bits text, difficulty text, chainwork text, n_tx text, previousblockhash text, nextblockhash text, data text, PRIMARY KEY (hash, height)) WITH CLUSTERING ORDER BY (height DESC);"
)
session.execute(
"CREATE TABLE IF NOT EXISTS bitcoin.transactions (hash text, version text, size text, vsize text, weight text, locktime text, vin text, vout text, block_height text, PRIMARY KEY (hash, block_height)) WITH CLUSTERING ORDER BY (block_height DESC);"
)
session.execute(
"CREATE TABLE IF NOT EXISTS bitcoin.relation (block_hash text PRIMARY KEY, transaction_hashes text );"
)
from cassandra.cluster import Cluster
from datetime import datetime
import pika
import os
import binascii
RABBITMQ_HOST = os.environ.get('RABBITMQ_HOST', 'rabbitmq')
RABBITMQ_PORT = int(os.environ.get('RABBITMQ_PORT', '5672'))
RABBITMQ_USERNAME = os.environ.get('RABBITMQ_USERNAME', 'loader_raw')
RABBITMQ_PASSWORD = os.environ.get('RABBITMQ_PASSWORD', 'waggle')
CASSANDRA_HOSTS = os.environ.get('CASSANDRA_HOSTS', 'cassandra').split()
BEEHIVE_DEPLOYMENT = os.environ.get('BEEHIVE_DEPLOYMENT', '/')
cluster = Cluster(contact_points=CASSANDRA_HOSTS)
session = cluster.connect('waggle')
query = 'INSERT INTO sensor_data_raw (node_id, date, plugin_name, plugin_version, plugin_instance, timestamp, parameter, data) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
prepared = session.prepare(query)
def process_message(ch, method, properties, body):
versionStrings = properties.app_id.split(':')
sampleDatetime = datetime.utcfromtimestamp(
float(properties.timestamp) / 1000.0)
sampleDate = sampleDatetime.strftime('%Y-%m-%d')
# TODO Validate / santize node_id here.
node_id = properties.reply_to
plugin_name = versionStrings[0]
plugin_version = versionStrings[1]
plugin_instance = '0' if (len(versionStrings) < 3) else versionStrings[2]
timestamp = int(properties.timestamp)
parameter = properties.type
def token_aware(self, keyspace, use_prepared=False):
use_singledc()
cluster = Cluster(
load_balancing_policy=TokenAwarePolicy(RoundRobinPolicy()),
protocol_version=PROTOCOL_VERSION)
session = cluster.connect()
wait_for_up(cluster, 1, wait=False)
wait_for_up(cluster, 2, wait=False)
wait_for_up(cluster, 3)
create_schema(session, keyspace, replication_factor=1)
self._insert(session, keyspace)
self._query(session, keyspace, use_prepared=use_prepared)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 12)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
self.coordinator_stats.reset_counts()
self._query(session, keyspace, use_prepared=use_prepared)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 12)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
self.coordinator_stats.reset_counts()
force_stop(2)
wait_for_down(cluster, 2, wait=True)
try:
self._query(session, keyspace, use_prepared=use_prepared)
self.fail()
except Unavailable as e:
self.assertEqual(e.consistency, 1)
self.assertEqual(e.required_replicas, 1)
self.assertEqual(e.alive_replicas, 0)
self.coordinator_stats.reset_counts()
start(2)
wait_for_up(cluster, 2, wait=True)
self._query(session, keyspace, use_prepared=use_prepared)
self.coordinator_stats.assert_query_count_equals(self, 1, 0)
self.coordinator_stats.assert_query_count_equals(self, 2, 12)
self.coordinator_stats.assert_query_count_equals(self, 3, 0)
self.coordinator_stats.reset_counts()
stop(2)
wait_for_down(cluster, 2, wait=True)
try:
self._query(session, keyspace, use_prepared=use_prepared)
self.fail()
except Unavailable:
pass
self.coordinator_stats.reset_counts()
start(2)
wait_for_up(cluster, 2, wait=True)
decommission(2)
wait_for_down(cluster, 2, wait=True)
self._query(session, keyspace, use_prepared=use_prepared)
results = set([
self.coordinator_stats.get_query_count(1),
self.coordinator_stats.get_query_count(3)
])
self.assertEqual(results, set([0, 12]))
self.coordinator_stats.assert_query_count_equals(self, 2, 0)
cluster.shutdown()
def __init__(self, config, logger):
self.config = config
self.logger = logger
cluster = Cluster(self.config.cassandra_cluster)
self.session = cluster.connect(self.config.cassandra_keyspace)
def __init__(self):
config = configparser.ConfigParser()
config.read('config.cfg')
self.cluster = Cluster([config.get('ClusterConn', 'cluster')])
self.session = self.cluster.connect(
config.get('ClusterConn', 'keyspace'))
from uuid import uuid4
from datetime import datetime
from cassandra.cluster import Cluster
from cassandra.query import BatchStatement
cluster = Cluster(['127.0.0.1'])
session = cluster.connect('dbbuilder')
insert_file_command = session.prepare(
"INSERT INTO files (id, name, created_on, columns) VALUES (?, ?, ?, ?)")
insert_files_by_username_command = session.prepare(
"INSERT INTO files_by_username (username, file_id, file_name) VALUES (?, ?, ?)"
)
insert_file_batch = BatchStatement()
username = '******'
file_id = uuid4()
file_name = 'test.xlsx'
creation_date = datetime.now()
columns = {}
insert_file_batch.add(insert_file_command,
(file_id, file_name, creation_date, columns))
insert_file_batch.add(insert_files_by_username_command,
(username, file_id, file_name))
session.execute(insert_file_batch)
def init(self, address='127.0.0.1', port=9042):
self._cluster = Cluster(contact_points=[address], port=port)
self._session = self._cluster.connect()
self._init_key_space()
self._init_table()
# In[1]:
#cuando no se tiene instalado una libreria en python se usa el comando pip pero para hacer la instalacion global se puede hacer con el comando !pip
get_ipython().system('pip install cassandra-driver')
# In[2]:
#importamos esa libreria su clase cluster
from cassandra.cluster import Cluster
# In[3]:
#conectamos nuestro codigo python con la cassandra que esta corriendo en el contenedor docker
#recordemos que podemos conectarnos a mas nodos delimitando distintas ips por ','
cluster = Cluster(contact_points=['192.168.0.16'], port=9042)
session = cluster.connect()
# ### Leemos el fichero con pandas
# In[5]:
#importamos y renombramos por cuestiones de buenas practicas
import pandas as pd
import numpy as np
# In[7]:
#leemos nuestro fichero csv
#especificamos la ruta del fichero (para saber nuestra ruta podemos ejecutar el comando !ls similar a linux)
suicides = pd.read_csv('./suicidios.csv', sep=',', thousands=b',', header=0)
KEYSPACE = "zeus"
NODE_TABLE = "node"
EDGE_TABLE = "edge"
# By default, takes localhost
# For actual, specify all the machine ip as a list
# cluster = Cluster(['192.168.0.1', '192.168.0.2'])
# The set of IP addresses we pass to the Cluster is
# simply an initial set of contact points. After the
# driver connects to one of these nodes it will
# automatically discover the rest of the nodes in the
# cluster and connect to them, so you don't need to
# list every node in your cluster.
print "Connecting with the cassandra cluster"
cluster = Cluster(['54.218.56.39'])
session = cluster.connect(KEYSPACE)
print "Connected"
print
def get_uuid():
return uuid.uuid1()
def get_column_names(table):
rows = session.execute(
"""
SELECT column_name FROM system_schema.columns
WHERE table_name=%s AND keyspace_name=%s;
""", (table, KEYSPACE))
"""CRAYFIS Cassandra Database
"""
# get server IP address
import docker
__client = docker.from_env()
__server = __client.containers.get('crayvault')
__ipaddr = __server.attrs['NetworkSettings']['IPAddress']
# connect to the server
from cassandra.cluster import Cluster
__cluster = Cluster([__ipaddr])
def get_session():
return __cluster.connect()
#!/usr/bin/python3
import sys
from cassandra import ConsistencyLevel # type: ignore
from cassandra.query import SimpleStatement # type: ignore
from cassandra.cluster import Cluster # type: ignore
ip = str(sys.argv[1])
sid = str(sys.argv[2])
if sid.startswith('0x'):
sid = sid[2:]
c = Cluster([ip])
s = c.connect()
res = s.execute(SimpleStatement('select * from system_distributed.cdc_streams', consistency_level=ConsistencyLevel.QUORUM))
res = sorted(res, key=lambda r: r[0])
for r in res:
print(r[0], len(r[2]))
for r in res:
if sid in (s.hex() for s in r[2]):
print(sid, 'is in', r[0])
break
client = pulsar.Client('pulsar://pulsar:6650')
consumer = client.subscribe(topic='covid19',subscription_name='covid19-to-astra3',schema=AvroSchema(Covid19))
# Connect to Astra
import os
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
astra_user = os.environ.get('ASTRA_USER')
cloud_config = {
'secure_connect_bundle': '/usr/src/app/secure-connect-'+astra_user+'.zip'
}
auth_provider = PlainTextAuthProvider(username=astra_user, password=os.environ.get('ASTRA_PASS'))
cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
session = cluster.connect()
session = cluster.connect('zekedean')
# Process the data in pulsar
while True:
msg = consumer.receive()
try:
astra_load = msg.value()
astra_load = [astra_load.date, astra_load.confirmed, astra_load.deaths, astra_load.recovered, astra_load.country]
print(astra_load)
session.execute("INSERT INTO covid19 (date, confirmed, deaths, recovered, country ) VALUES (%s,%s,%s,%s,%s)", astra_load)
#print(msg.value())
# Acknowledge successful processing of the message
consumer.acknowledge(msg)
except:
def __init__(self, cluser_IP=['ec2-18-233-215-146.compute-1.amazonaws.com']):
cluster = Cluster(cluser_IP)
self.session = cluster.connect()
self.keyspace_name = None
self.table_name = None
###############################################################################
if __name__ == '__main__':
args = ParseCommandLine()
response = requests.get('https://scheduler.monroe-system.eu/v1/resources', cert=(args.certificate, args.privateKey))
if response.status_code != 200:
# This means something went wrong.
print('GET /v1/resources {}'.format(response.status_code))
print('Response Headers: {}'.format(response.headers))
sys.exit()
authProvider = PlainTextAuthProvider(username='******', password='******')
cluster = Cluster(['127.0.0.1'], 9042, auth_provider = authProvider, connect_timeout = 15)
session = cluster.connect('monroe')
runningStart = args.startDateTime
while True:
runningEnd = datetime(runningStart.year, runningStart.month, runningStart.day + 1)
if runningEnd > args.endDateTime:
runningEnd = args.endDateTime
epoch = datetime.utcfromtimestamp(0)
runningStartTimeStamp = int((runningStart - epoch).total_seconds());
runningEndTimeStamp = int((runningEnd - epoch).total_seconds());
noGpsDataCount = 0
