def refresh_connection(self,
host=None,
port=None,
cql_version="2.0.0",
num_retries=10):
"""
establish a connection to the server. retry if needed.
"""
if host:
self._host = host
if port:
self._port = port
self._cql_version = cql_version or None
for try_num in xrange(1 + num_retries):
try:
if self._cql_version:
con = cql.connect(self._host,
self._port,
keyspace=None,
cql_version=self._cql_version)
else:
con = cql.connect(self._host, self._port, keyspace=None)
self._cached_cursor = con.cursor()
if self._is_keyspace_created:
cql_str = "USE %s" % self._params['keyspace_name']
self.execute_query(cql_str)
except:
if try_num < num_retries:
time.sleep(1)
else:
raise
def refresh_connection(self, host=None, port=None, cql_version="2.0.0", num_retries=10):
"""
establish a connection to the server. retry if needed.
"""
if host:
self._host = host
if port:
self._port = port
self._cql_version = cql_version or None
for try_num in xrange(1+num_retries):
try:
if self._cql_version:
con = cql.connect(self._host, self._port, keyspace=None, cql_version=self._cql_version)
else:
con = cql.connect(self._host, self._port, keyspace=None)
self._cached_cursor = con.cursor()
if self._is_keyspace_created:
cql_str = "USE %s" % self._params['keyspace_name']
self.execute_query(cql_str)
except:
if try_num < num_retries:
time.sleep(1)
else:
raise
def cql_connection(self,
node,
keyspace=None,
version=None,
user=None,
password=None):
import cql
host, port = node.network_interfaces['thrift']
if not version and self.cluster.version() >= "1.2":
version = "3.0.0"
elif not version and self.cluster.version() >= "1.1":
version = "2.0.0"
if version:
con = cql.connect(host,
port,
keyspace=keyspace,
cql_version=version,
user=user,
password=password)
else:
con = cql.connect(host,
port,
keyspace=keyspace,
user=user,
password=password)
self.connections.append(con)
return con
def cql_connection(self, node, keyspace=None, version=None):
import cql
host, port = node.network_interfaces['thrift']
if version:
con = cql.connect(host, port, keyspace=keyspace, cql_version=version)
else:
con = cql.connect(host, port, keyspace=keyspace)
self.connections.append(con)
return con
def test_connecting_with_keyspace(self):
# this conn is just for creating the keyspace
conn = cql.connect(TEST_HOST, TEST_NATIVE_PORT, native=True)
curs = conn.cursor()
with self.with_keyspace(curs, conn.cql_version) as ksname:
curs.execute('create table blah1_%s (a int primary key, b int);'
% self.randstr)
conn2 = cql.connect(TEST_HOST, TEST_NATIVE_PORT, keyspace=ksname,
native=True, cql_version=conn.cql_version)
curs2 = conn2.cursor()
curs2.execute('select * from blah1_%s;' % self.randstr)
conn2.close()
def cql_connection(self, node, keyspace=None, version=None):
import cql
host, port = node.network_interfaces["thrift"]
if not version and self.cluster.version() >= "1.2":
version = "3.0.0"
if version:
con = cql.connect(host, port, keyspace=keyspace, cql_version=version)
else:
con = cql.connect(host, port, keyspace=keyspace)
self.connections.append(con)
return con
def cql_connection(self, node, keyspace=None, version=None, user=None, password=None):
import cql
host, port = node.network_interfaces['thrift']
if not version and self.cluster.version() >= "1.2":
version = "3.0.0"
elif not version and self.cluster.version() >= "1.1":
version = "2.0.0"
if version:
con = cql.connect(host, port, keyspace=keyspace, cql_version=version, user=user, password=password)
else:
con = cql.connect(host, port, keyspace=keyspace, user=user, password=password)
self.connections.append(con)
return con
def dump_archives(archives):
name = path.replace('/opt/graphite/storage/whisper/', '', 1)
name = name.replace('.wsp', '', 1)
name = name.replace('/', '.')
con = cql.connect('127.0.0.1', 9160, 'metric', cql_version='3.0.0')
print("Connected to Cassandra!")
cursor = con.cursor()
for i, archive in enumerate(archives):
print 'Archive %d data:' % i
offset = archive['offset']
for point in xrange(archive['points']):
(timestamp,
value) = struct.unpack(whisper.pointFormat,
map[offset:offset + whisper.pointSize])
print '%d: %d, %10.35g' % (point, timestamp, value)
offset += whisper.pointSize
period = archive['retention']
rollup = archive['secondsPerPoint']
ttl = period
#CQLString = "UPDATE metric USING TTL ? SET data = data + ? WHERE tenant = '' AND rollup = ? AND period = ? AND path = ? AND time = ?;"
#cursor.execute(CQLString, [ttl, value, rollup, period, name, timestamp])
#print CQLString, [ttl, value, rollup, period, name, timestamp]
if timestamp > 0:
CQLString = "UPDATE metric USING TTL %d SET data = data + [ %d ] WHERE tenant = '' AND rollup = %d AND period = %d AND path = '%s' AND time = %d;" % (
ttl, value, rollup, period, name, timestamp)
#print CQLString
cursor.execute(CQLString)
print
def __init__(self, config):
if not cql:
raise ImportError("cql is required to use Cassandra as storage.")
self.name = 'cassandra'
self.storage = cql.connect(config["host"], config["port"], cql_version='3.0.0')
cursor = self.storage.cursor()
cursor.execute("""USE %s""" % config["keyspace"])
def cql_execute(host, port, filename, force, silent=False):
try:
connection = cql.connect(host=host, port=port, cql_version='3.0.0')
except thrift.transport.TTransport.TTransportException, e:
if not silent:
print 'Execution failed: {e}'.format(e=e)
return False
def _create_connection(self):
"""
Creates a new connection for the connection pool.
should only return a valid connection that it's actually connected to
"""
if not self.hosts:
raise Exception("At least one host required")
random.shuffle(self.hosts)
for hoststr in self.hosts:
host, port = hoststr.split(':')
try:
new_conn = cql.connect(host,
int(port),
user=self.username,
password=self.password,
consistency_level=self.consistency)
new_conn.set_cql_version('3.0.0')
return new_conn
except Exception as e:
logging.exception(
"Could not establish connection to {}:{}".format(
host, port))
raise Exception("Could not connect to any server in cluster")
def __init__(self):
dbconn = cql.connect(config_local.cassandra_host, config_local.cassandra_port )
cursor = dbconn.cursor()
try:
cursor.execute("USE " + config_global.cassandra_default_keyspace)
cql_query = """ DROP COLUMNFAMILY :rawJsonDump """
cursor.execute(cql_query, dict(rawJsonDump = config_global.dbname_rawJson))
except cql.ProgrammingError as programmingError:
print cql_query
print programmingError
try:
cursor.execute("USE " + config_global.cassandra_default_keyspace)
cursor.execute(""" CREATE COLUMNFAMILY :rawJsonDump
( incomingTime timestamp PRIMARY KEY, jsonString text )
WITH comment = ' simply the raw input data received from the contest server ' """,
dict(rawJsonDump = config_global.dbname_rawJson))
except cql.ProgrammingError as programmingError:
print programmingError
""" lets try getting the latest items ordered"""
cursor.execute("USE " + config_global.cassandra_default_keyspace)
cursor.execute(""" SELECT FIRST 5 * FROM :rawJsonDump LIMIT 5""", dict(rawJsonDump = config_global.dbname_rawJson))
row = cursor.fetchone()
while (row):
print row
row = cursor.fetchone()
def __init__(self):
dbconn = cql.connect(config_local.cassandra_host, config_local.cassandra_port)
cursor = dbconn.cursor()
try:
cursor.execute(""" USE :keyspace """, dict(keyspace = config_global.cassandra_default_keyspace))
except cql.ProgrammingError as programmingError:
print programmingError
try:
cursor.execute(""" DROP COLUMNFAMILY :columnfamily; """, dict(columnfamily = config_global.dbname_dimensionList))
pass
except cql.ProgrammingError as programmingError:
print programmingError
try:
cursor.execute("""
CREATE COLUMNFAMILY :columnfamily
(
dimension_name text PRIMARY KEY
)
WITH comparator = int AND default_validation = int;
""",
dict(columnfamily = config_global.dbname_dimensionList))
except cql.ProgrammingError as programmingError:
print programmingError
""" lets insert row keys """
for dimension in config_global.dbname_dimensionList_rowKeys:
try:
for dimension_value in xrange(3):
cursor.execute("""INSERT INTO :table ( dimension_name, :dimension_value ) VALUES
( :dimension_name, :dimension_value) USING TTL 10 """,
dict(table=config_global.dbname_dimensionList,
dimension_name = dimension,
dimension_value = dimension_value)
)
except cql.ProgrammingError as programmingError:
print programmingError
""" now we have to check if a dimension is already known """
try:
for test_id in xrange(1):
cursor.execute("""SELECT FIRST 3 REVERSED * FROM :table WHERE dimension_name = :dimension_name""",
dict(table=config_global.dbname_dimensionList,
dimension_name = dimension
)
)
print cursor.fetchall()
except cql.ProgrammingError as programmingError:
print programmingError
def _create_connection(self):
"""
Creates a new connection for the connection pool.
should only return a valid connection that it's actually connected to
"""
if not self.hosts:
raise Exception("At least one host required")
random.shuffle(self.hosts)
for hoststr in self.hosts:
host, port = hoststr.split(':')
try:
new_conn = cql.connect(
host,
int(port),
user=self.username,
password=self.password,
consistency_level=self.consistency
)
new_conn.set_cql_version('3.0.0')
return new_conn
except Exception as e:
logging.exception("Could not establish connection to {}:{}".format(host, port))
raise Exception("Could not connect to any server in cluster")
def testSynteticDataGet_filter_b(self):
print "filter_b"
dbconn2 = cql.connect(config_local.cassandra_host, config_local.cassandra_port )
cursorb = dbconn2.cursor()
cursorb.execute("USE plistaContest")
startTime = time.time()
count = 300
for n in xrange(count):
filter_b_val = int( random.uniform(1,100) )
cursorb.execute("""SELECT * FROM itemTableFull WHERE
filter_b = :filter_b_val
LIMIT 4""",
dict(filter_b_val = filter_b_val))
resultset = cursorb.fetchall()
endTime = time.time() - startTime
print "endTime: \t" + str(endTime)
print "requests per seconds:\t" + str ( count / endTime)
resultset = cursorb.fetchall()
#print resultset
#print len(resultset)
print "\n\n"
def cql_execute(host, port, filename, force, silent=False):
try:
connection = cql.connect(host=host, port=port, cql_version='3.0.0')
except thrift.transport.TTransport.TTransportException, e:
if not silent:
print 'Execution failed: {e}'.format(e=e)
return False
def _create_connection(self):
"""
Creates a new connection for the connection pool.
should only return a valid connection that it's actually connected to
"""
if not self._hosts:
raise CQLConnectionError("At least one host required")
hosts = copy(self._hosts)
random.shuffle(hosts)
for host in hosts:
try:
new_conn = cql.connect(host.name,
host.port,
user=self._username,
password=self._password,
consistency_level=self._consistency)
new_conn.set_cql_version('3.0.0')
return new_conn
except Exception as e:
logging.debug("Could not establish connection to {}:{}".format(
host.name, host.port))
pass
raise CQLConnectionError("Could not connect to any server in cluster")
def truncateCf():
con = cql.connect('10.1.2.91', 9160, "apixio", cql_version='3.0.0')
print("Connected!")
cursor = con.cursor()
CQLString = "TRUNCATE store457;"
#cursor.execute(CQLString)
pass
def _create_connection(self):
"""
Creates a new connection for the connection pool.
should only return a valid connection that it's actually connected to
"""
if not self._hosts:
raise CQLConnectionError("At least one host required")
hosts = copy(self._hosts)
random.shuffle(hosts)
for host in hosts:
try:
transport = self._create_transport(host)
new_conn = cql.connect(
host.name,
host.port,
user=self._username,
password=self._password,
consistency_level=self._consistency,
transport=transport
)
new_conn.set_cql_version('3.0.0')
return new_conn
except Exception as e:
logging.debug("Could not establish connection to {}:{}".format(host.name, host.port))
pass
raise CQLConnectionError("Could not connect to any server in cluster")
def status(name):
try:
conn = connect(host=cassandra_server, port=cassandra_port)
cursor = conn.cursor()
cql_command = "USE {0}".format(name)
cursor.execute(cql_command)
except (ProgrammingError, TTransportException), e:
return e.message, 500
def validate_schema_consistent(self, node):
""" Makes sure that there is only one schema """
host, port = node.network_interfaces['thrift']
conn = cql.connect(host, port, keyspace=None)
schemas = conn.client.describe_schema_versions()
num_schemas = len(schemas)
assert num_schemas == 1, "There were multiple schema versions: " + pprint.pformat(schemas)
def setUpModule():
'''Remove the side effects of previous modules in homer'''
try:
global connection
connection = cql.connect("localhost", 9160).cursor()
connection.execute("DROP KEYSPACE Test;")
except Exception as e:
print e
def get_connection(self, host, port=9160):
""" get connection to specified host """
conn = cql.connect(host,
port,
user=self.username,
password=self.password,
consistency_level='ONE')
conn.set_cql_version('3.0.0')
return conn
def validate_schema_consistent(self, node):
""" Makes sure that there is only one schema """
debug("validate_schema_consistent() " + node.name)
host, port = node.network_interfaces['thrift']
conn = cql.connect(host, port, keyspace=None)
schemas = conn.client.describe_schema_versions()
num_schemas = len([ss for ss in schemas.keys() if ss != 'UNREACHABLE'])
assert num_schemas == 1, "There were multiple schema versions: " + pprint.pformat(schemas)
def connect(self):
"""Connect to the two databases
self.connection is for NGram Corpus on Cassandra
self.connection2 is for UMLS on SQLServer 2008
"""
## This is the cassandra database with the 4-gram Google Web Corpus
self.connection = cql.connect('127.0.0.1', 9160, "fourgm", cql_version = '3.0.0')
self.cursor = self.connection.cursor()
def connect(self):
"""Connect to the two databases
self.connection is for NGram Corpus on Cassandra
self.connection2 is for UMLS on SQLServer 2008
"""
self.connection = cql.connect('127.0.0.1', 9160, "fourgm", cql_version = '3.0.0')
self.cursor = self.connection.cursor()
self.connection2 = pyodbc.connect('DRIVER={SQL Server};SERVER=XXX.XXX.XXX.XXX;DATABASE=umlsSmall;UID=XXXXXX;PWD=XXXXXXXXX')
self.cursor2=self.connection2.cursor()
def test_execution_fails_after_close(self):
conn = cql.connect(TEST_HOST, TEST_NATIVE_PORT, native=True)
curs = conn.cursor()
with self.with_keyspace(curs, conn.cql_version) as ksname:
curs.execute('use "%s"' % ksname)
curs.execute('create table blah (a int primary key, b int);')
curs.execute('select * from blah;')
conn.close()
self.assertRaises(cql.ProgrammingError, curs.execute,
'select * from blah;')
def validate_schema_consistent(self, node):
""" Makes sure that there is only one schema """
debug("validate_schema_consistent() " + node.name)
host, port = node.network_interfaces['thrift']
conn = cql.connect(host, port, keyspace=None)
schemas = conn.client.describe_schema_versions()
num_schemas = len([ss for ss in schemas.keys() if ss != 'UNREACHABLE'])
assert num_schemas == 1, "There were multiple schema versions: " + pprint.pformat(
schemas)
def setUp(self):
try:
self.dbconn = cql.connect(TEST_HOST, TEST_PORT, cql_version=TEST_CQL_VERSION)
except cql.thrifteries.TApplicationException:
# set_cql_version (and thus, cql3) not supported; skip all of these
self.cursor = None
return
self.cursor = self.dbconn.cursor()
self.keyspace = self.create_schema()
def __init__(self):
dbconn = cql.connect(config_local.cassandra_host, config_local.cassandra_port)
cursor = dbconn.cursor()
try:
cursor.execute(""" USE :keyspace """, dict(keyspace = config_global.cassandra_default_keyspace))
except cql.ProgrammingError as programmingError:
print programmingError
try:
cursor.execute(""" DROP COLUMNFAMILY :columnfamily; """, dict(columnfamily = config_global.dbname_recommendationsList))
pass
except cql.ProgrammingError as programmingError:
print programmingError
try:
cursor.execute("""
CREATE COLUMNFAMILY :columnfamily
(
user_id text PRIMARY KEY
)
WITH comparator = int AND default_validation = int;
""",
dict(columnfamily = config_global.dbname_recommendationsList))
except cql.ProgrammingError as programmingError:
print programmingError
""" lets insert row keys """
for user_id in xrange(5):
for item_id in xrange(5):
try:
cursor.execute("""INSERT INTO :table ( user_id, :item_id ) VALUES
( :user_id, :score ) USING TTL 10 """,
dict(table = config_global.dbname_recommendationsList,
user_id = user_id,
item_id = item_id,
score = 5 - item_id )
)
except cql.ProgrammingError as programmingError:
print programmingError
try:
cursor.execute("""SELECT FIRST 3 REVERSED * FROM :table WHERE user_id = 2""",
dict(table=config_global.dbname_recommendationsList
)
)
print cursor.fetchone()
except cql.ProgrammingError as programmingError:
print programmingError
def get_or_create():
connection = get()
if connection == None:
_log.info("Connecting to Cassandra on %s", settings.CASS_HOST)
connection = cql.connect(settings.CASS_HOST,
settings.CASS_PORT,
settings.CASS_KEYSPACE,
cql_version='2.0.0')
assert connection
thread_local.connection = connection
return connection
def __create_connection(self):
return cql.connect(
choice(self.hostname),
port=self.port,
keyspace=self.keyspace,
user=self.user,
password=self.password,
cql_version=self.cql_version,
compression=self.compression,
consistency_level=self.consistency_level,
transport=self.transport)
def connect(self):
"""Connect to the two databases
self.connection is for NGram Corpus on Cassandra
self.connection2 is for UMLS on SQLServer 2008
"""
##self.connection = pyodbc.connect('DRIVER={SQL Server};Trusted_Connection=yes;SERVER=WIN-20IR78KE8UV\SQLSERVER2012;DATABASE=GoogleCorpus;UID=Administrator')
##self.cursor = self.connection.cursor()
## This is the cassandra database with the 4-gram Google Web Corpus
self.connection = cql.connect('127.0.0.1', 9160, "fourgm", cql_version = '3.0.0')
self.cursor = self.connection.cursor()
def get_or_create():
connection = get()
if connection == None:
_log.info("Connecting to Cassandra on %s", settings.CASS_HOST)
connection = cql.connect(settings.CASS_HOST,
settings.CASS_PORT,
settings.CASS_KEYSPACE,
cql_version='2.0.0')
assert connection
thread_local.connection = connection
return connection
def standard_connection(self):
"""Set up a connection to Cassandra.
"""
logging.debug('connecting to %s', self.option_group.keyspace)
self.conn = cql.connect(self.option_group.hostname,
self.option_group.port,
self.option_group.keyspace,
user=self.option_group.username,
password=self.option_group.password,
cql_version=self.option_group.cqlversion)
return
def standard_connection(self):
"""Set up a connection to Cassandra.
"""
logging.debug('connecting to %s', self.option_group.keyspace)
self.conn = cql.connect(self.option_group.hostname,
self.option_group.port,
self.option_group.keyspace,
user=self.option_group.username,
password=self.option_group.password,
cql_version=self.option_group.cqlversion)
return
def test_connecting_with_compression(self):
try:
import snappy
except ImportError:
if hasattr(unittest, 'skipTest'):
unittest.skipTest('Snappy compression not available')
else:
return
conn = cql.connect(TEST_HOST, TEST_NATIVE_PORT, native=True,
compression=True)
self.assertEqual(conn.compressor, snappy.compress)
self.try_basic_stuff(conn)
def connect_cql():
con = None
cursor = None
try:
con = cql.connect('localhost', 9160, 'fav_test', cql_version='3.0.0')
print("Connected!")
cursor = con.cursor()
yield cursor
except Exception, e:
print "Error occurred: " + str(e)
def my_view(request):
now = datetime.datetime.now()
fp = open('/Users/Del/Desktop/mushrooms/mushrooms/media/mushroomsWeb/index.html')
t = Template(fp.read())
fp.close()
con = cql.connect('localhost', 9160,'demo', cql_version='3.0.0')
cursor = con.cursor()
#CQLString = "create columnfamily t1 (okey int primary key, observer varchar, image varchar)"
#x=cursor.execute(CQLString)
html = t.render(Context({'current_date': now}))
return HttpResponse(html)
def get_or_create(keyspace):
connection = thread_local.connections.get(keyspace)
if not connection:
_log.info("Connecting to Cassandra on %s", settings.CASS_HOST)
connection = cql.connect(settings.CASS_HOST,
settings.CASS_PORT,
keyspace,
cql_version='2.0.0')
assert connection
thread_local.connections[keyspace] = connection
return connection
def connect_cql():
con = None
cursor = None
try:
con = cql.connect('localhost', 9160, 'fav_test', cql_version='3.0.0')
print ("Connected!")
cursor = con.cursor()
yield cursor;
except Exception, e:
print "Error occurred: " + str(e)
def __init__(self, host, port, keyspace):
try:
self.host = host
self.port = port
self.keyspace = keyspace
self.transport = tfactory.regular_transport_factory('127.0.0.1', 9160, None, None)
self.con = cql.connect('127.0.0.1', 9160, cql_version='3', transport=self.transport)
self.cursor = self.con.cursor()
self.cursor.execute('USE ' + self.keyspace + ' ;');
except cql.ProgrammingError as e:
print e.args
sys.exit(1)
def add_instance():
keyspace = None
if request.method == 'POST' and 'name' in request.form:
keyspace = request.form.get('name')
try:
conn = connect(host=cassandra_server, port=cassandra_port)
cursor = conn.cursor()
cql_command = "CREATE KEYSPACE {0} WITH strategy_class = 'SimpleStrategy' AND strategy_options:replication_factor=3;"
cursor.execute(cql_command.format(keyspace))
except (ProgrammingError, TTransportException), e:
return e.message, 500
return "", 201
def connect_to_cassandra():
"""
Connect to Cassandra host
"""
conn = cql.connect(host=c.host,
port=c.port,
keyspace=c.keyspace,
user=c.user,
password=c.password,
cql_version=c.cql_version)
logging.info("cassandra_lib connected")
return conn
def url_parse(self, url, **kwargs):
location = urlparse(url)
if location.scheme in ('cassandra','cql'):
host = location.netloc or "localhost:9160"
if re.search(":[0-9]+$", host):
ip,port = host.split(':')
else:
ip = host
port = 9160
keyspace = location.path[1:] or kwargs.get('database', 'kairos')
if '?' in keyspace:
keyspace,params = keyspace.split('?')
return cql.connect(ip, int(port), keyspace, cql_version='3.0.0', **kwargs)
def exequteCQL(self, query):
# establish the cql connection
if self.cql_cursor == None:
self.cql_conn = cql.connect(self.host, self.port, self.keyspace)
self.cql_cursor = self.cql_conn.cursor()
topics = self.getAllTopics()
# rename topics to column names ...
for topic in topics:
query = query.replace('"' + topic + '"',
'"' + self.topic2Hash(topic) + '"')
print query
#return query
# execute querie
self.cql_cursor.execute(query)
return self.cql_cursor.fetchall()
def connect(keyspace=None, use_cache=True, user=None, password=None):
key = (keyspace, user)
if keyspace is None:
keyspace = KEYSPACE
if use_cache and pool.has_key(key):
return pool[key]
for i in range(len(NODES)):
try:
node = get_node()
con = cql.connect(node,
keyspace=keyspace,
cql_version='3.0.0',
user=user,
password=password)
print "Connected to %s" % node
pool[key] = con
return con
except Exception, msg:
print msg # TODO -- logger
def _connection(self):
'''
Return a connection from the pool
'''
try:
return self._pool.get(False)
except Empty:
args = [self._host, self._port, self._keyspace]
kwargs = {
'user': None,
'password': None,
'cql_version': self._cql_version,
'compression': self._compression,
'consistency_level': self._consistency_level,
'transport': self._transport,
}
if self._credentials:
kwargs['user'] = self._credentials['user']
kwargs['password'] = self._credentials['password']
return cql.connect(*args, **kwargs)
def create_schema(self):
"""Creates the schema if it doesn't exist using the CQL in self.SCHEMA.
Each query in there will be formatted with locals(), so if you
update self.SCHEMA, be sure to update this function, too.
"""
logging.info('creating schema')
self.conn = cql.connect(self.option_group.hostname,
self.option_group.port,
"system",
user=self.option_group.username,
password=self.option_group.password,
cql_version=self.option_group.cqlversion)
data_center = self.query_or_die(
self.SELECT_ALL_DATACENTERS,
"Unable to determine the local data center")
if not data_center:
logging.fatal(
"No peers defined, repairs on a single-node cluster are silly")
exit(0)
# Cassandra doesn't support 'SELECT foo, 1 FROM ..." or DISTINCT,
# so we have do something a little complicated to deduplicate the
# results and then produce the desired string.
data_center_replication_map = {}
for row in data_center:
data_center_replication_map[row[0]] = None
data_center_replication_map = ", ".join(
["'%s':3" % x for x in data_center_replication_map])
# This declaration is just so that "keyspace" will appear in locals.
# pylint: disable=unused-variable
keyspace = self.option_group.keyspace
# pylint: enable=unused-variable
for cql_query in self.SCHEMA:
self.query(cql_query.format(**locals()))
return
def query():
try:
con = cql.connect(host='114.215.156.15',
keyspace="hw2_1452739",
cql_version='3.0.0')
cursor = con.cursor()
cursor.execute("select * from test")
list = cursor.result
#cluster = Cluster(['114.215.156.15', ])
#session = cluster.connect()
#session.execute("use hw2_1452739;")
#cqlString = "SELECT * from test"
# #list = session.execute(cqlString)
answerArray = analysis(list)
for each in answerArray:
'''print with out enter'''
print each,
except Exception, exce:
print '!!!!!'
print exce
def functional_test(self):
NUM_SUBCOLS = 100
NUM_ADDS = 100
cluster = self.cluster
cluster.populate(3).start()
node1 = cluster.nodelist()[0]
time.sleep(.5)
cursor = self.patient_cql_connection(node1).cursor()
self.create_ks(cursor, 'ks', 3)
time.sleep(1) # wait for propagation
# create the columnfamily using thrift
host, port = node1.network_interfaces['thrift']
thrift_conn = cql.connect(host, port, keyspace='ks')
cf_def = CfDef(keyspace='ks',
name='cf',
column_type='Super',
default_validation_class='CounterColumnType')
thrift_conn.client.system_add_column_family(cf_def)
# let the sediment settle to to the bottom before drinking...
time.sleep(2)
for subcol in xrange(NUM_SUBCOLS):
for add in xrange(NUM_ADDS):
column_parent = ColumnParent(column_family='cf',
super_column='subcol_%d' % subcol)
counter_column = CounterColumn('col_0', 1)
thrift_conn.client.add('row_0', column_parent, counter_column,
ConsistencyLevel.QUORUM)
time.sleep(1)
# flush everything and the problem will be mostly corrected.
# for node in cluster.nodelist():
# node.flush()
debug("Stopping cluster")
cluster.stop()
time.sleep(5)
debug("Starting cluster")
cluster.start()
time.sleep(5)
thrift_conn = cql.connect(host, port, keyspace='ks')
from_db = []
for i in xrange(NUM_SUBCOLS):
column_path = ColumnPath(column_family='cf',
column='col_0',
super_column='subcol_%d' % i)
column_or_super_column = thrift_conn.client.get(
'row_0', column_path, ConsistencyLevel.QUORUM)
val = column_or_super_column.counter_column.value
debug(str(val)),
from_db.append(val)
debug("")
expected = [NUM_ADDS for i in xrange(NUM_SUBCOLS)]
if from_db != expected:
raise Exception(
"Expected a bunch of the same values out of the db. Got this: "
+ str(from_db))
h = resp.headers
h['Access-Control-Allow-Origin'] = origin
h['Access-Control-Allow-Methods'] = get_methods()
h['Access-Control-Max-Age'] = str(max_age)
if headers is not None:
h['Access-Control-Allow-Headers'] = headers
return resp
f.provide_automatic_options = False
return update_wrapper(wrapped_function, f)
return decorator
# connect to database
con = cql.connect('ec2-54-187-166-118.us-west-2.compute.amazonaws.com',
'9160', 'janusz_forex_rt_demo', cql_version='3.1.1')
cursor = con.cursor()
tasks = {
'AUDUSD':0,
'EURUSD':0,
'GBPUSD':0,
'NZDUSD':0,
'USDCAD':0,
'USDCHF':0,
'USDJPY':0
}
pairs = ['AUDUSD', 'EURUSD', 'GBPUSD', 'NZDUSD', 'USDCAD', 'USDCHF', 'USDJPY']
q = "select issued_at, ask, bid from ticks where pair_day = :key and issued_at > :utc"
def main(argv):
usage = "Usage: dbcql {query}"
args, kwargs = parse(argv)
if len(args) > 1:
error(output, "Unexpected argument: '%s'" % args[1], 2)
if len(args) < 1:
error(output, "No query argument", 2)
keyspace = kwargs.get('keyspace', None)
host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)
# UNDONE: credentials ..
query = args[0]
if query is None:
error(output, "Command requires a single query argument", 2)
# A query may consist of multiple expressions. We execute each of
# the expressions in order and output the results from the final
# expression. The primary scenario is:
#
# "USE {keyspace}; SELECT * FROM .."
#
# However, arbitrary query expressions may be combined in this way.
expressions = query.split(';')
# Scan the expressions looking for anything we need to disable
for expression in expressions:
# UNDONE: The following disables usage of the CLQ DROP command
# in order to prevent users from inadvertently (or maliciously)
# dropping of critical keyspace data until we can properly enable
# the capability by integrating with Splunk's role based access
# control (where this would be an admin-only capability).
if re.match("\s*drop ", expression, re.I):
error(output, "The CQL DROP command has been disabled.", 2)
connection = None
cursor = None
try:
connection = cql.connect(host, port, keyspace)
cursor = connection.cursor()
# Everything looks ok .. so run them.
for expression in expressions:
cursor.execute(expression)
# A 'SELECT *' expression requires special handling because the
# results may be "ragged", meaning that we don't know what fields
# will appear for any given row. In order to handle this case we
# scan an initial `batchsize` set of rows, collecting all fields
# that we see. We then use those fields for the results header and
# collect any additional fields that we subsequently see into an
# "_extra" MV field. Note that a row in Cassandra can contain up
# to 2B columns, so "SELECT *" may have other issues aside from the
# incrimental processing impact on this search script.
if re.search("select\s+\*", expression, re.I):
ragged(cursor)
else:
normal(cursor)
except:
error(output, excinfo(), 2)
finally:
if cursor is not None:
cursor.close()
if connection is not None:
connection.close()
def setUp(self):
self.client = cql.connect('localhost', 9160, os.environ.get('CASSANDRA_KEYSPACE','kairos'), cql_version='3.0.0')
super(CassandraSetTest,self).setUp()
import sys
import os
import csv
from datetime import datetime
import time
import cql
# writefile = open('CSULander_Modified.txt', 'w')
con = cql.connect('localhost', cql_version='3.0.0')
print("Connected!")
with open(sys.argv[1], 'r') as data:
mycsv = csv.reader(data)
# Writing the Header File to the output file
# writefile.write('Date,TimeBucket,TimeStamp,FromIP,ToIP,PortNumber,DataIn,DataOut\n')
for row in mycsv:
day = 1
month = 12
year = 2014
timeBucket_H = 0
timeBucket_M = 0
time = 0
in_pos = 2
out_pos = 291
import json
import cql
conn = cql.connect(host="127.0.0.1", port=9160, keyspace="sensordb", cql_version='5.0.1')
cur=conn.cursor()
#===============================================================
# Functions to push Sensor Data into Database
# Function to save Temperature to DB Table
def DHT11_Temp_Data_Handler(jsonData):
#Parse Data
json_Dict = json.loads(jsonData)
SensorID = json_Dict['Sensor_ID']
Data_and_Time = json_Dict['Date']
Temperature = json_Dict['Temperature']
#Push into DB Table
cur.execute("insert into temperature (SensorID, Date_n_Time, Temperature) VALUES ('SensorID','Data_and_Time','Temperature')")
cur.close()
print "Inserted Temperature Data into Database."
print ""
# Function to save Humidity to DB Table
def DHT11_Humidity_Data_Handler(jsonData):
#Parse Data
json_Dict = json.loads(jsonData)
SensorID = json_Dict['Sensor_ID']
Data_and_Time = json_Dict['Date']
Humidity = json_Dict['Humidity']