def open(self):
server_list = [self.host + ":" + self.port]
connected = False
try:
if self.username and self.password:
credentials = {
'username': self.username,
'password': self.password
}
self.pool = ConnectionPool(keyspace=self.keyspace,
server_list=server_list,
credentials=credentials,
pool_size=self.pool_size)
if self.pool:
self.logged_in = True
connected = True
else:
self.pool = ConnectionPool(keyspace=self.keyspace,
server_list=server_list,
pool_size=self.pool_size)
if self.pool:
connected = True
except Exception, e:
pass
def __init__(self, namespace, server_list=['localhost:9160']):
# save cassandra server
self.server_list = server_list
self.namespace = namespace
self._closed = False
#setup_logging(self)
# Connect to the server creating the namespace if it doesn't
# already exist
try:
self.pool = ConnectionPool(namespace,
self.server_list,
max_retries=500,
pool_timeout=600,
timeout=10)
except pycassa.InvalidRequestException:
self._create_namespace(namespace)
self.pool = ConnectionPool(namespace,
self.server_list,
max_retries=500,
pool_timeout=600,
timeout=10)
try:
self._tasks = pycassa.ColumnFamily(self.pool, 'tasks')
except pycassa.NotFoundException:
self._create_column_family('tasks',
key_validation_class=ASCII_TYPE,
bytes_columns=['task_data'])
self._tasks = pycassa.ColumnFamily(self.pool, 'tasks')
try:
self._available = pycassa.ColumnFamily(self.pool, 'available')
except pycassa.NotFoundException:
self._create_column_family('available',
key_validation_class=ASCII_TYPE,
bytes_columns=['available'])
self._available = pycassa.ColumnFamily(self.pool, 'available')
try:
self._task_count = pycassa.ColumnFamily(self.pool, 'task_count')
except pycassa.NotFoundException:
self._create_counter_column_family('task_count',
key_validation_class=ASCII_TYPE,
counter_columns=['task_count'])
self._task_count = pycassa.ColumnFamily(self.pool, 'task_count')
self._task_count.insert('RowKey', {'task_count': 0})
try:
self._available_count = pycassa.ColumnFamily(
self.pool, 'available_count')
except pycassa.NotFoundException:
self._create_counter_column_family(
'available_count',
key_validation_class=ASCII_TYPE,
counter_columns=['available_count'])
self._available_count = pycassa.ColumnFamily(
self.pool, 'available_count')
self._available_count.insert('RowKey', {'available_count': 0})
def __init__(self, client, qname):
super(Processor, self).__init__(client, qname)
self.pool = ConnectionPool('processing_llama_Processor')
self.trends = ColumnFamily(self.pool, 'Trend')
def get_sleep_time():
return 60
def _update_analytics_start_time(self, start_time):
if mockcassandra.use_cql():
cluster = Cluster(['127.0.0.1'],
port=int(self.__class__.cassandra_port))
session = cluster.connect(COLLECTOR_KEYSPACE_CQL)
query = "INSERT INTO {0} (key, \"{1}\") VALUES ('{2}', {3})".format(
SYSTEM_OBJECT_TABLE, SYSTEM_OBJECT_START_TIME,
SYSTEM_OBJECT_ANALYTICS, start_time)
try:
session.execute(query)
except Exception as e:
logging.error("INSERT INTO %s: Key %s Column %s Value %d "
"FAILED: %s" %
(SYSTEM_OBJECT_TABLE, SYSTEM_OBJECT_ANALYTICS,
SYSTEM_OBJECT_START_TIME, start_time, str(e)))
assert False
else:
cluster.shutdown()
else:
pool = ConnectionPool(
COLLECTOR_KEYSPACE,
['127.0.0.1:%s' % (self.__class__.cassandra_port)])
col_family = ColumnFamily(pool, SYSTEM_OBJECT_TABLE)
col_family.insert(SYSTEM_OBJECT_ANALYTICS,
{SYSTEM_OBJECT_START_TIME: start_time})
def test_caching_pre_11(self):
version = tuple(
[int(v) for v in sys._conn.describe_version().split('.')])
if version >= (19, 30, 0):
raise SkipTest('CF specific caching no longer supported.')
sys.create_column_family(TEST_KS,
'CachedCF10',
row_cache_size=100,
key_cache_size=100,
row_cache_save_period_in_seconds=3,
key_cache_save_period_in_seconds=3)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'CachedCF10')
assert_equal(cf._cfdef.row_cache_size, 100)
assert_equal(cf._cfdef.key_cache_size, 100)
assert_equal(cf._cfdef.row_cache_save_period_in_seconds, 3)
assert_equal(cf._cfdef.key_cache_save_period_in_seconds, 3)
sys.alter_column_family(TEST_KS,
'CachedCF10',
row_cache_size=200,
key_cache_size=200,
row_cache_save_period_in_seconds=4,
key_cache_save_period_in_seconds=4)
cf1 = ColumnFamily(pool, 'CachedCF10')
assert_equal(cf1._cfdef.row_cache_size, 200)
assert_equal(cf1._cfdef.key_cache_size, 200)
assert_equal(cf1._cfdef.row_cache_save_period_in_seconds, 4)
assert_equal(cf1._cfdef.key_cache_save_period_in_seconds, 4)
def get_cassandra_connection(keyspace_name, hosts):
key = keyspace_name, tuple(hosts)
connection_pool, created_at = connection_pool_cache.get(key, (None, None))
init_new_pool = connection_pool is None or connection_pool_expired(
created_at)
if connection_pool is not None and len(connection_pool.server_list) == 0:
logging.error('connection pool had no active hosts')
init_new_pool = True
if init_new_pool:
nodes = detect_nodes(hosts, keyspace_name)
logger.info('setting up a new connection pool')
connection_pool = ConnectionPool(
keyspace_name,
nodes,
pool_size=settings.FEEDLY_CASSANDRA_CONNECTION_POOL_SIZE,
prefill=False,
timeout=settings.FEEDLY_CASSANDRA_TIMEOUT,
max_retries=3)
listener = FeedlyPoolListener(connection_pool)
connection_pool.add_listener(listener)
connection_pool_cache[key] = (connection_pool, time.time())
return connection_pool
def connect_server(address, key_space, column_family):
"""Establish connection."""
pool = ConnectionPool(key_space, address)
print "[INFO] Connection to '" + key_space + "' established."
cf = ColumnFamily(pool, column_family)
print "[INFO] Column family '" + column_family + "' used."
return pool, cf
def __init__(self, client, qname, trend=5):
super(Buyer, self).__init__(client, uuid.uuid4().hex)
self.holdings = {}
self.cash = 100000.0
self.history = {}
self.trend = trend
self.pool = ConnectionPool('example_consumer_Buyer')
self.stored_holdings = ColumnFamily(self.pool, 'Holdings')
self.quote_history = ColumnFamily(self.pool, 'Quotes')
self.stored_cash = ColumnFamily(self.pool, 'Cash')
try:
cash = self.stored_cash.get('current')
self.cash = cash['amount']
except ttypes.NotFoundException:
self.stored_cash.insert('current', {'amount': self.cash})
for symbol, columns in self.stored_holdings.get_range():
self.holdings[symbol] = (columns['number_of_shares'],
columns['price'], columns['cost'])
date_expression = create_index_expression('timestamp',
datetime.date.today(), GT)
date_clause = create_index_clause([date_expression], count=1000)
for key, columns in self.quote_history.get_range():
symbol = columns['symbol']
price = columns['price']
self.add_quote(symbol, price)
def _update_analytics_start_time(self, start_time):
pool = ConnectionPool(
COLLECTOR_KEYSPACE,
['127.0.0.1:%s' % (self.__class__.cassandra_port)])
col_family = ColumnFamily(pool, SYSTEM_OBJECT_TABLE)
col_family.insert(SYSTEM_OBJECT_ANALYTICS,
{SYSTEM_OBJECT_START_TIME: start_time})
def main():
default_db_host = socket.gethostname()
default_db_port = "9160"
default_key_space = "ContrailAnalytics"
default_table_space = "FlowRecordTable"
try:
(opts, args) = getopt.getopt(sys.argv[1:], "n:p:k:t:sh", \
["summary", "help"])
except getopt.GetoptError:
usage()
show_summary = False
for o, a in opts:
# (k, v) = opt
if o in ["-h", "--help"]:
usage()
elif o in ["-n", "--host"]:
default_db_host = a
elif o in ["-p", "--port"]:
default_db_port = a
elif o in ["-k", "--key"]:
default_key_space = a
elif o in ["-t", "--table"]:
default_table_space = a
elif o in ["-s", "--summary"]:
show_summary = True
try:
level = 1
pool = ConnectionPool(
keyspace=default_key_space,
server_list=[default_db_host + ":" + default_db_port],
timeout=0.5,
max_retries=2)
f = pycassa.ColumnFamily(pool, default_table_space)
row_count = sum(1 for _ in f.get_range())
if (row_count >= 0):
print "HTTP/1.1 200 OK"
print "Content-Type: Content-Type: text/plain"
print
print "ColumnFamily: %s in KeySpace:%s has %d rows" % (
default_table_space, default_key_space, row_count)
print
print "Cassandra checked!"
sys.exit(0)
else:
print "HTTP/1.1 500 Internal Server Error"
print "Content-Type: Content-Type: text/plain"
print
print "Unable to query ColumnFamily %s" % (default_table_space)
sys.exit(1)
except Exception as e:
print "HTTP/1.1 503 Service Unavailable"
print "Content-Type: Content-Type: text/plain"
print
print str(e)
sys.exit(2)
def setUp(self):
n = 10000
self.weibo_ids = self._load_items(n)
pool = ConnectionPool(
'master_timeline',
server_list=['219.224.135.60:9160', '219.224.135.61:9160'],
pool_size=10)
col_fam = pycassa.ColumnFamily(pool, 'weibos')
self.weibos_col_fam = col_fam
def do_use(self, keyspace):
if keyspace not in self.sm.list_keyspaces():
return self.perror('Unknown keyspace %s' % keyspace)
self.prompt = '%s/%s> ' % (self.server, keyspace)
self.keyspace = keyspace
self.pool = ConnectionPool(keyspace, server_list=[self.server])
print 'Using %s as default keyspace' % keyspace
def __init__(self, keyspace=None, host='localhost', port='9160'):
self.host = host
self.port = port
self.keyspace = keyspace
self.pool = ConnectionPool(self.keyspace,
[self.host + ':' + self.port])
self.providerCF = pycassa.ColumnFamily(self.pool, self.providerCFname)
self.patientCF = pycassa.ColumnFamily(self.pool, self.patientCFname)
self.patientMapCF = pycassa.ColumnFamily(self.pool,
self.patientMapCFname)
def __init__(self, keyspacename, serverport):
self.serverport = serverport
self.keyspace = keyspacename
self.sys = SystemManager(self.serverport)
self.cfs = self.sys.get_keyspace_column_families(self.keyspace)
self.keyspacenames = self.cfs.keys()
self.pool = ConnectionPool(self.keyspace, [ self.serverport ])
self.column_family = {}
self.createtables()
self.reinit_column_family_map()
def verify_with_thrift(self):
# No more thrift in 4.0
if self.cluster.version() >= '4':
return
pool = ConnectionPool("supcols", pool_size=1)
super_col_fam = ColumnFamily(pool, "cols")
for name in NAMES:
super_col_value = super_col_fam.get(name)
self.assertEqual(OrderedDict([(('attr', u'name'), name)]),
super_col_value)
def connect_db(self):
try:
self._pool = ConnectionPool(
COLLECTOR_KEYSPACE,
server_list=self._cassandra_server_list,
timeout=None)
except Exception as e:
self._logger.error("Exception: Failure in connection to "
"AnalyticsDb %s" % e)
return -1
return None
def setUp(self):
self.sysman = SystemManager()
self.sysman.create_keyspace(
TEST_KS, SIMPLE_STRATEGY,
dict(replication_factor='1')
)
self.sysman.create_column_family(TEST_KS, TEST_CF)
self.pool = ConnectionPool(TEST_KS)
self.cf = ColumnFamily(self.pool, TEST_CF)
def feeder(start_point_dict):
frontend = {}
backend = {}
try:
frontend['pool'] = ConnectionPool(F_KS, F_SERVLIST)
frontend['handle'] = ColumnFamily(frontend['pool'], F_CF)
backend['pool'] = ConnectionPool(B_KS, B_SERVLIST)
backend['handle'] = ColumnFamily(backend['pool'], B_CF)
except pycassa.NotFoundException as err:
print "[ERROR] " + str(err)
exit(-1)
except pycassa.AllServersUnavailable as err:
print "[ERROR] " + str(err)
exit(-1)
counter = 0
for sp in start_point_dict:
filelist = get_filelist(start_point_dict[sp])
if not filelist:
print "[WARN] No files were found."
continue
for filename in filelist:
if os.path.getsize(filename) > SIZE_LIMIT:
continue
file_content = ""
with open(filename, "rb") as f:
file_content = f.read()
hasherlist = [hashlib.md5(), hashlib.sha1()]
taskid = gen_taskid(hasherlist, filename, file_content)
print "%s\n%s\n%s" % (sp, filename, taskid)
frontend['handle'].insert(filename, {sp: taskid})
backend['handle'].insert(taskid, {'content': file_content})
print "Uploaded\n"
counter += 1
print "Total " + str(counter) + " file(s) uploaded."
frontend['pool'].dispose()
backend['pool'].dispose()
def verify_cassandra(thriftport, cqlport, cassandra_user, cassandra_password):
retry_threshold = 10
retry = 1
while retry < retry_threshold:
try:
if cassandra_user is None and cassandra_password is None:
pool = ConnectionPool(None, ['localhost:' + str(thriftport)])
else:
cred = {
'username': cassandra_user,
'password': cassandra_password
}
pool = ConnectionPool(None, ['localhost:'+str(thriftport)], \
credentials=cred)
return True
except Exception as e:
logging.info("Exception: Failure in connection to "
"AnalyticsDb: Retry %d: %s" % (retry, e))
retry = retry + 1
time.sleep(5)
return False
def test_alter_column_non_bytes_type(self):
sys.create_column_family(TEST_KS, 'LongCF', comparator_type=LONG_TYPE)
sys.create_index(TEST_KS, 'LongCF', 3, LONG_TYPE)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'LongCF')
cf.insert('key', {3: 3})
assert_equal(cf.get('key')[3], 3)
sys.alter_column(TEST_KS, 'LongCF', 2, LONG_TYPE)
cf = ColumnFamily(pool, 'LongCF')
cf.insert('key', {2: 2})
assert_equal(cf.get('key')[2], 2)
def test_alter_column_family_default_validation_class(self):
sys.create_column_family(TEST_KS,
'AlteredCF',
default_validation_class=LONG_TYPE)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'AlteredCF')
assert_equal(cf.default_validation_class, "LongType")
sys.alter_column_family(TEST_KS,
'AlteredCF',
default_validation_class=UTF8_TYPE)
cf = ColumnFamily(pool, 'AlteredCF')
assert_equal(cf.default_validation_class, "UTF8Type")
def column_family_remove(self, machine_id, keyspace_name,
column_family_name, key):
"""Remove a key from column family for a given keyspace """
if (self.keyspace_contains(keyspace_name,
column_family_name) == False):
print "Error : Keyspace:column family could not be found."
return False
pool = ConnectionPool(keyspace=keyspace_name,
server_list=keyspace.server_ips,
prefill=False)
col_fam = ColumnFamily(pool, column_family_name)
col_fam.remove(key)
return True
def atualiza_code():
import sys
sys.path.append('./pymongo')
sys.path.append('./pycassa')
import pymongo
import pycassa
from pycassa.pool import ConnectionPool
from pycassa import index
from pycassa.columnfamily import ColumnFamily
#
pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'],
timeout=10000000000)
#
tb_object1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_OBJECT')
tb_object_dt1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_OBJECT_DT')
tb_relaction1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_RELACTIONS')
tb_know1 = pycassa.ColumnFamily(pool2, 'knowledge_manager')
#
MONGO_URL = 'mongodb://*****:*****@ds061938.mongolab.com:61938/mdnet'
conn = pymongo.Connection(MONGO_URL)
dbM = conn.mdnet
#
tb_object = dbM['SEMANTIC_OBJECT']
tb_object_dt = dbM['SEMANTIC_OBJECT_DT']
tb_relaction = dbM['SEMANTIC_RELACTIONS']
tb_know = dbM['knowledge_manager']
#
tb_object.remove()
tb_object_dt.remove()
tb_relaction.remove()
tb_know.remove()
#
r1 = tb_object1.get_range()
for k, r in r1:
r['id'] = k
tb_object.insert(r)
#
r1 = tb_object_dt1.get_range()
for k, r in r1:
r['id'] = k
tb_object_dt.insert(r)
r1 = tb_relaction1.get_range()
for k, r in r1:
r['id'] = k
tb_relaction.insert(r)
#===
r1 = tb_know1.get_range()
for k, r in r1:
r['id'] = k
tb_know.insert(r)
def setUp(self):
self.keyspace = 'Claimspace'
self.host = 'localhost'
self.port = '9160'
self.pool = ConnectionPool(self.keyspace,
[self.host + ':' + self.port])
self.cf = pycassa.ColumnFamily(self.pool, Database.providerCFname)
self.db = Database(self.keyspace)
self.providerDict = {
'username': '******',
'realname': 'Testy User',
'providerid': '8675309'
}
self.providerObj = Provider(**self.providerDict)
def reinit_column_family_map(self):
self.cfs = self.sys.get_keyspace_column_families(self.keyspace)
self.keyspacenames = self.cfs.keys()
self.pool = ConnectionPool(self.keyspace, [ self.serverport ] )
self.column_family = {}
for ks in self.keyspacenames:
self.column_family[ks] = ColumnFamily(self.pool, ks)
self.column_family['SimpleTable']._set_key_validation_class(IntegerType())
self.column_family['Threads']._set_key_validation_class(IntegerType())
self.column_family['UserThreads']._set_key_validation_class(IntegerType())
self.column_family['ThreadObjects']._set_key_validation_class(IntegerType())
self.column_family['Objects']._set_key_validation_class(UUIDType())
self.column_family['ThreadHolds']._set_key_validation_class(IntegerType())
self.column_family['ObjectHistory']._set_key_validation_class(UUIDType())
def colum_family_insert(self, machine_id, keyspace_name,
column_family_name, user_content):
"""Insert into a column family for a given keyspace """
if (self.keyspace_contains(keyspace_name,
column_family_name) == False):
print "Error : Keyspace:column family could not be found."
return False
pool = ConnectionPool(keyspace=keyspace_name,
server_list=keyspace.server_ips,
prefill=False)
col_fam = ColumnFamily(pool, column_family_name)
for content in user_content:
col_fam.insert(content, user_content[content])
#col_fam.insert('Key2', {'name':'mayur', 'age':'23'})
return True
def start_socket():
global sock
global pool2
while True:
try:
pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'],
timeout=10000000000)
mdTb.start_db(pool2)
get_object.start_db(pool2)
get_object2.start_db(pool2)
Identify.start_db(pool2)
except Exception, e:
print 'Not connected(cassandra)....', e
time.sleep(5)
continue
break
def test_column_validators(self):
validators = {'name': UTF8_TYPE, 'age': LONG_TYPE}
sys.create_column_family(TEST_KS,
'ValidatedCF',
column_validation_classes=validators)
pool = ConnectionPool(TEST_KS)
cf = ColumnFamily(pool, 'ValidatedCF')
cf.insert('key', {'name': 'John', 'age': 40})
self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
validators = {'name': ASCII_TYPE, 'age': INT_TYPE}
sys.alter_column_family(TEST_KS,
'ValidatedCF',
column_validation_classes=validators)
cf.load_schema()
self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
def get_values(servlst, ks, cf, key):
#print key
try:
pool = ConnectionPool(ks, servlst)
cf_handle = ColumnFamily(pool, cf)
result = cf_handle.get(key).values()
except pycassa.NotFoundException as err:
print "[ERROR] " + key + " not found"
result = ""
except Exception as err:
print "[ERROR] " + str(err)
exit(-1)
finally:
pool.dispose()
return result
def init_base():
global pool2
global tb_fuzzy
global tb_fz_store_pref
global tb_fz_store_defs
global tb_fz_store_sufix
global tb_fz_store_refer
global tb_fz_arround_points
#
pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'], timeout=10000)
tb_fuzzy = pycassa.ColumnFamily(pool2, 'fuzzy_store')
tb_fz_store_pref = pycassa.ColumnFamily(pool2, "fz_store_pref")
tb_fz_store_defs = pycassa.ColumnFamily(pool2, "fz_store_defs")
tb_fz_store_sufix = pycassa.ColumnFamily(pool2, "fz_store_sufix")
tb_fz_store_refer = pycassa.ColumnFamily(pool2, "fz_store_refer")
tb_fz_arround_points = pycassa.ColumnFamily(pool2, "fz_arround_points")
