def open(self): server_list = [self.host + ":" + self.port] connected = False try: if self.username and self.password: credentials = { 'username': self.username, 'password': self.password } self.pool = ConnectionPool(keyspace=self.keyspace, server_list=server_list, credentials=credentials, pool_size=self.pool_size) if self.pool: self.logged_in = True connected = True else: self.pool = ConnectionPool(keyspace=self.keyspace, server_list=server_list, pool_size=self.pool_size) if self.pool: connected = True except Exception, e: pass
def __init__(self, namespace, server_list=['localhost:9160']): # save cassandra server self.server_list = server_list self.namespace = namespace self._closed = False #setup_logging(self) # Connect to the server creating the namespace if it doesn't # already exist try: self.pool = ConnectionPool(namespace, self.server_list, max_retries=500, pool_timeout=600, timeout=10) except pycassa.InvalidRequestException: self._create_namespace(namespace) self.pool = ConnectionPool(namespace, self.server_list, max_retries=500, pool_timeout=600, timeout=10) try: self._tasks = pycassa.ColumnFamily(self.pool, 'tasks') except pycassa.NotFoundException: self._create_column_family('tasks', key_validation_class=ASCII_TYPE, bytes_columns=['task_data']) self._tasks = pycassa.ColumnFamily(self.pool, 'tasks') try: self._available = pycassa.ColumnFamily(self.pool, 'available') except pycassa.NotFoundException: self._create_column_family('available', key_validation_class=ASCII_TYPE, bytes_columns=['available']) self._available = pycassa.ColumnFamily(self.pool, 'available') try: self._task_count = pycassa.ColumnFamily(self.pool, 'task_count') except pycassa.NotFoundException: self._create_counter_column_family('task_count', key_validation_class=ASCII_TYPE, counter_columns=['task_count']) self._task_count = pycassa.ColumnFamily(self.pool, 'task_count') self._task_count.insert('RowKey', {'task_count': 0}) try: self._available_count = pycassa.ColumnFamily( self.pool, 'available_count') except pycassa.NotFoundException: self._create_counter_column_family( 'available_count', key_validation_class=ASCII_TYPE, counter_columns=['available_count']) self._available_count = pycassa.ColumnFamily( self.pool, 'available_count') self._available_count.insert('RowKey', {'available_count': 0})
def __init__(self, client, qname): super(Processor, self).__init__(client, qname) self.pool = ConnectionPool('processing_llama_Processor') self.trends = ColumnFamily(self.pool, 'Trend') def get_sleep_time(): return 60
def _update_analytics_start_time(self, start_time): if mockcassandra.use_cql(): cluster = Cluster(['127.0.0.1'], port=int(self.__class__.cassandra_port)) session = cluster.connect(COLLECTOR_KEYSPACE_CQL) query = "INSERT INTO {0} (key, \"{1}\") VALUES ('{2}', {3})".format( SYSTEM_OBJECT_TABLE, SYSTEM_OBJECT_START_TIME, SYSTEM_OBJECT_ANALYTICS, start_time) try: session.execute(query) except Exception as e: logging.error("INSERT INTO %s: Key %s Column %s Value %d " "FAILED: %s" % (SYSTEM_OBJECT_TABLE, SYSTEM_OBJECT_ANALYTICS, SYSTEM_OBJECT_START_TIME, start_time, str(e))) assert False else: cluster.shutdown() else: pool = ConnectionPool( COLLECTOR_KEYSPACE, ['127.0.0.1:%s' % (self.__class__.cassandra_port)]) col_family = ColumnFamily(pool, SYSTEM_OBJECT_TABLE) col_family.insert(SYSTEM_OBJECT_ANALYTICS, {SYSTEM_OBJECT_START_TIME: start_time})
def test_caching_pre_11(self): version = tuple( [int(v) for v in sys._conn.describe_version().split('.')]) if version >= (19, 30, 0): raise SkipTest('CF specific caching no longer supported.') sys.create_column_family(TEST_KS, 'CachedCF10', row_cache_size=100, key_cache_size=100, row_cache_save_period_in_seconds=3, key_cache_save_period_in_seconds=3) pool = ConnectionPool(TEST_KS) cf = ColumnFamily(pool, 'CachedCF10') assert_equal(cf._cfdef.row_cache_size, 100) assert_equal(cf._cfdef.key_cache_size, 100) assert_equal(cf._cfdef.row_cache_save_period_in_seconds, 3) assert_equal(cf._cfdef.key_cache_save_period_in_seconds, 3) sys.alter_column_family(TEST_KS, 'CachedCF10', row_cache_size=200, key_cache_size=200, row_cache_save_period_in_seconds=4, key_cache_save_period_in_seconds=4) cf1 = ColumnFamily(pool, 'CachedCF10') assert_equal(cf1._cfdef.row_cache_size, 200) assert_equal(cf1._cfdef.key_cache_size, 200) assert_equal(cf1._cfdef.row_cache_save_period_in_seconds, 4) assert_equal(cf1._cfdef.key_cache_save_period_in_seconds, 4)
def get_cassandra_connection(keyspace_name, hosts): key = keyspace_name, tuple(hosts) connection_pool, created_at = connection_pool_cache.get(key, (None, None)) init_new_pool = connection_pool is None or connection_pool_expired( created_at) if connection_pool is not None and len(connection_pool.server_list) == 0: logging.error('connection pool had no active hosts') init_new_pool = True if init_new_pool: nodes = detect_nodes(hosts, keyspace_name) logger.info('setting up a new connection pool') connection_pool = ConnectionPool( keyspace_name, nodes, pool_size=settings.FEEDLY_CASSANDRA_CONNECTION_POOL_SIZE, prefill=False, timeout=settings.FEEDLY_CASSANDRA_TIMEOUT, max_retries=3) listener = FeedlyPoolListener(connection_pool) connection_pool.add_listener(listener) connection_pool_cache[key] = (connection_pool, time.time()) return connection_pool
def connect_server(address, key_space, column_family): """Establish connection.""" pool = ConnectionPool(key_space, address) print "[INFO] Connection to '" + key_space + "' established." cf = ColumnFamily(pool, column_family) print "[INFO] Column family '" + column_family + "' used." return pool, cf
def __init__(self, client, qname, trend=5): super(Buyer, self).__init__(client, uuid.uuid4().hex) self.holdings = {} self.cash = 100000.0 self.history = {} self.trend = trend self.pool = ConnectionPool('example_consumer_Buyer') self.stored_holdings = ColumnFamily(self.pool, 'Holdings') self.quote_history = ColumnFamily(self.pool, 'Quotes') self.stored_cash = ColumnFamily(self.pool, 'Cash') try: cash = self.stored_cash.get('current') self.cash = cash['amount'] except ttypes.NotFoundException: self.stored_cash.insert('current', {'amount': self.cash}) for symbol, columns in self.stored_holdings.get_range(): self.holdings[symbol] = (columns['number_of_shares'], columns['price'], columns['cost']) date_expression = create_index_expression('timestamp', datetime.date.today(), GT) date_clause = create_index_clause([date_expression], count=1000) for key, columns in self.quote_history.get_range(): symbol = columns['symbol'] price = columns['price'] self.add_quote(symbol, price)
def _update_analytics_start_time(self, start_time): pool = ConnectionPool( COLLECTOR_KEYSPACE, ['127.0.0.1:%s' % (self.__class__.cassandra_port)]) col_family = ColumnFamily(pool, SYSTEM_OBJECT_TABLE) col_family.insert(SYSTEM_OBJECT_ANALYTICS, {SYSTEM_OBJECT_START_TIME: start_time})
def main(): default_db_host = socket.gethostname() default_db_port = "9160" default_key_space = "ContrailAnalytics" default_table_space = "FlowRecordTable" try: (opts, args) = getopt.getopt(sys.argv[1:], "n:p:k:t:sh", \ ["summary", "help"]) except getopt.GetoptError: usage() show_summary = False for o, a in opts: # (k, v) = opt if o in ["-h", "--help"]: usage() elif o in ["-n", "--host"]: default_db_host = a elif o in ["-p", "--port"]: default_db_port = a elif o in ["-k", "--key"]: default_key_space = a elif o in ["-t", "--table"]: default_table_space = a elif o in ["-s", "--summary"]: show_summary = True try: level = 1 pool = ConnectionPool( keyspace=default_key_space, server_list=[default_db_host + ":" + default_db_port], timeout=0.5, max_retries=2) f = pycassa.ColumnFamily(pool, default_table_space) row_count = sum(1 for _ in f.get_range()) if (row_count >= 0): print "HTTP/1.1 200 OK" print "Content-Type: Content-Type: text/plain" print print "ColumnFamily: %s in KeySpace:%s has %d rows" % ( default_table_space, default_key_space, row_count) print print "Cassandra checked!" sys.exit(0) else: print "HTTP/1.1 500 Internal Server Error" print "Content-Type: Content-Type: text/plain" print print "Unable to query ColumnFamily %s" % (default_table_space) sys.exit(1) except Exception as e: print "HTTP/1.1 503 Service Unavailable" print "Content-Type: Content-Type: text/plain" print print str(e) sys.exit(2)
def setUp(self): n = 10000 self.weibo_ids = self._load_items(n) pool = ConnectionPool( 'master_timeline', server_list=['219.224.135.60:9160', '219.224.135.61:9160'], pool_size=10) col_fam = pycassa.ColumnFamily(pool, 'weibos') self.weibos_col_fam = col_fam
def do_use(self, keyspace): if keyspace not in self.sm.list_keyspaces(): return self.perror('Unknown keyspace %s' % keyspace) self.prompt = '%s/%s> ' % (self.server, keyspace) self.keyspace = keyspace self.pool = ConnectionPool(keyspace, server_list=[self.server]) print 'Using %s as default keyspace' % keyspace
def __init__(self, keyspace=None, host='localhost', port='9160'): self.host = host self.port = port self.keyspace = keyspace self.pool = ConnectionPool(self.keyspace, [self.host + ':' + self.port]) self.providerCF = pycassa.ColumnFamily(self.pool, self.providerCFname) self.patientCF = pycassa.ColumnFamily(self.pool, self.patientCFname) self.patientMapCF = pycassa.ColumnFamily(self.pool, self.patientMapCFname)
def __init__(self, keyspacename, serverport): self.serverport = serverport self.keyspace = keyspacename self.sys = SystemManager(self.serverport) self.cfs = self.sys.get_keyspace_column_families(self.keyspace) self.keyspacenames = self.cfs.keys() self.pool = ConnectionPool(self.keyspace, [ self.serverport ]) self.column_family = {} self.createtables() self.reinit_column_family_map()
def verify_with_thrift(self): # No more thrift in 4.0 if self.cluster.version() >= '4': return pool = ConnectionPool("supcols", pool_size=1) super_col_fam = ColumnFamily(pool, "cols") for name in NAMES: super_col_value = super_col_fam.get(name) self.assertEqual(OrderedDict([(('attr', u'name'), name)]), super_col_value)
def connect_db(self): try: self._pool = ConnectionPool( COLLECTOR_KEYSPACE, server_list=self._cassandra_server_list, timeout=None) except Exception as e: self._logger.error("Exception: Failure in connection to " "AnalyticsDb %s" % e) return -1 return None
def setUp(self): self.sysman = SystemManager() self.sysman.create_keyspace( TEST_KS, SIMPLE_STRATEGY, dict(replication_factor='1') ) self.sysman.create_column_family(TEST_KS, TEST_CF) self.pool = ConnectionPool(TEST_KS) self.cf = ColumnFamily(self.pool, TEST_CF)
def feeder(start_point_dict): frontend = {} backend = {} try: frontend['pool'] = ConnectionPool(F_KS, F_SERVLIST) frontend['handle'] = ColumnFamily(frontend['pool'], F_CF) backend['pool'] = ConnectionPool(B_KS, B_SERVLIST) backend['handle'] = ColumnFamily(backend['pool'], B_CF) except pycassa.NotFoundException as err: print "[ERROR] " + str(err) exit(-1) except pycassa.AllServersUnavailable as err: print "[ERROR] " + str(err) exit(-1) counter = 0 for sp in start_point_dict: filelist = get_filelist(start_point_dict[sp]) if not filelist: print "[WARN] No files were found." continue for filename in filelist: if os.path.getsize(filename) > SIZE_LIMIT: continue file_content = "" with open(filename, "rb") as f: file_content = f.read() hasherlist = [hashlib.md5(), hashlib.sha1()] taskid = gen_taskid(hasherlist, filename, file_content) print "%s\n%s\n%s" % (sp, filename, taskid) frontend['handle'].insert(filename, {sp: taskid}) backend['handle'].insert(taskid, {'content': file_content}) print "Uploaded\n" counter += 1 print "Total " + str(counter) + " file(s) uploaded." frontend['pool'].dispose() backend['pool'].dispose()
def verify_cassandra(thriftport, cqlport, cassandra_user, cassandra_password): retry_threshold = 10 retry = 1 while retry < retry_threshold: try: if cassandra_user is None and cassandra_password is None: pool = ConnectionPool(None, ['localhost:' + str(thriftport)]) else: cred = { 'username': cassandra_user, 'password': cassandra_password } pool = ConnectionPool(None, ['localhost:'+str(thriftport)], \ credentials=cred) return True except Exception as e: logging.info("Exception: Failure in connection to " "AnalyticsDb: Retry %d: %s" % (retry, e)) retry = retry + 1 time.sleep(5) return False
def test_alter_column_non_bytes_type(self): sys.create_column_family(TEST_KS, 'LongCF', comparator_type=LONG_TYPE) sys.create_index(TEST_KS, 'LongCF', 3, LONG_TYPE) pool = ConnectionPool(TEST_KS) cf = ColumnFamily(pool, 'LongCF') cf.insert('key', {3: 3}) assert_equal(cf.get('key')[3], 3) sys.alter_column(TEST_KS, 'LongCF', 2, LONG_TYPE) cf = ColumnFamily(pool, 'LongCF') cf.insert('key', {2: 2}) assert_equal(cf.get('key')[2], 2)
def test_alter_column_family_default_validation_class(self): sys.create_column_family(TEST_KS, 'AlteredCF', default_validation_class=LONG_TYPE) pool = ConnectionPool(TEST_KS) cf = ColumnFamily(pool, 'AlteredCF') assert_equal(cf.default_validation_class, "LongType") sys.alter_column_family(TEST_KS, 'AlteredCF', default_validation_class=UTF8_TYPE) cf = ColumnFamily(pool, 'AlteredCF') assert_equal(cf.default_validation_class, "UTF8Type")
def column_family_remove(self, machine_id, keyspace_name, column_family_name, key): """Remove a key from column family for a given keyspace """ if (self.keyspace_contains(keyspace_name, column_family_name) == False): print "Error : Keyspace:column family could not be found." return False pool = ConnectionPool(keyspace=keyspace_name, server_list=keyspace.server_ips, prefill=False) col_fam = ColumnFamily(pool, column_family_name) col_fam.remove(key) return True
def atualiza_code(): import sys sys.path.append('./pymongo') sys.path.append('./pycassa') import pymongo import pycassa from pycassa.pool import ConnectionPool from pycassa import index from pycassa.columnfamily import ColumnFamily # pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'], timeout=10000000000) # tb_object1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_OBJECT') tb_object_dt1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_OBJECT_DT') tb_relaction1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_RELACTIONS') tb_know1 = pycassa.ColumnFamily(pool2, 'knowledge_manager') # MONGO_URL = 'mongodb://*****:*****@ds061938.mongolab.com:61938/mdnet' conn = pymongo.Connection(MONGO_URL) dbM = conn.mdnet # tb_object = dbM['SEMANTIC_OBJECT'] tb_object_dt = dbM['SEMANTIC_OBJECT_DT'] tb_relaction = dbM['SEMANTIC_RELACTIONS'] tb_know = dbM['knowledge_manager'] # tb_object.remove() tb_object_dt.remove() tb_relaction.remove() tb_know.remove() # r1 = tb_object1.get_range() for k, r in r1: r['id'] = k tb_object.insert(r) # r1 = tb_object_dt1.get_range() for k, r in r1: r['id'] = k tb_object_dt.insert(r) r1 = tb_relaction1.get_range() for k, r in r1: r['id'] = k tb_relaction.insert(r) #=== r1 = tb_know1.get_range() for k, r in r1: r['id'] = k tb_know.insert(r)
def setUp(self): self.keyspace = 'Claimspace' self.host = 'localhost' self.port = '9160' self.pool = ConnectionPool(self.keyspace, [self.host + ':' + self.port]) self.cf = pycassa.ColumnFamily(self.pool, Database.providerCFname) self.db = Database(self.keyspace) self.providerDict = { 'username': '******', 'realname': 'Testy User', 'providerid': '8675309' } self.providerObj = Provider(**self.providerDict)
def reinit_column_family_map(self): self.cfs = self.sys.get_keyspace_column_families(self.keyspace) self.keyspacenames = self.cfs.keys() self.pool = ConnectionPool(self.keyspace, [ self.serverport ] ) self.column_family = {} for ks in self.keyspacenames: self.column_family[ks] = ColumnFamily(self.pool, ks) self.column_family['SimpleTable']._set_key_validation_class(IntegerType()) self.column_family['Threads']._set_key_validation_class(IntegerType()) self.column_family['UserThreads']._set_key_validation_class(IntegerType()) self.column_family['ThreadObjects']._set_key_validation_class(IntegerType()) self.column_family['Objects']._set_key_validation_class(UUIDType()) self.column_family['ThreadHolds']._set_key_validation_class(IntegerType()) self.column_family['ObjectHistory']._set_key_validation_class(UUIDType())
def colum_family_insert(self, machine_id, keyspace_name, column_family_name, user_content): """Insert into a column family for a given keyspace """ if (self.keyspace_contains(keyspace_name, column_family_name) == False): print "Error : Keyspace:column family could not be found." return False pool = ConnectionPool(keyspace=keyspace_name, server_list=keyspace.server_ips, prefill=False) col_fam = ColumnFamily(pool, column_family_name) for content in user_content: col_fam.insert(content, user_content[content]) #col_fam.insert('Key2', {'name':'mayur', 'age':'23'}) return True
def start_socket(): global sock global pool2 while True: try: pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'], timeout=10000000000) mdTb.start_db(pool2) get_object.start_db(pool2) get_object2.start_db(pool2) Identify.start_db(pool2) except Exception, e: print 'Not connected(cassandra)....', e time.sleep(5) continue break
def test_column_validators(self): validators = {'name': UTF8_TYPE, 'age': LONG_TYPE} sys.create_column_family(TEST_KS, 'ValidatedCF', column_validation_classes=validators) pool = ConnectionPool(TEST_KS) cf = ColumnFamily(pool, 'ValidatedCF') cf.insert('key', {'name': 'John', 'age': 40}) self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40}) validators = {'name': ASCII_TYPE, 'age': INT_TYPE} sys.alter_column_family(TEST_KS, 'ValidatedCF', column_validation_classes=validators) cf.load_schema() self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
def get_values(servlst, ks, cf, key): #print key try: pool = ConnectionPool(ks, servlst) cf_handle = ColumnFamily(pool, cf) result = cf_handle.get(key).values() except pycassa.NotFoundException as err: print "[ERROR] " + key + " not found" result = "" except Exception as err: print "[ERROR] " + str(err) exit(-1) finally: pool.dispose() return result
def init_base(): global pool2 global tb_fuzzy global tb_fz_store_pref global tb_fz_store_defs global tb_fz_store_sufix global tb_fz_store_refer global tb_fz_arround_points # pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'], timeout=10000) tb_fuzzy = pycassa.ColumnFamily(pool2, 'fuzzy_store') tb_fz_store_pref = pycassa.ColumnFamily(pool2, "fz_store_pref") tb_fz_store_defs = pycassa.ColumnFamily(pool2, "fz_store_defs") tb_fz_store_sufix = pycassa.ColumnFamily(pool2, "fz_store_sufix") tb_fz_store_refer = pycassa.ColumnFamily(pool2, "fz_store_refer") tb_fz_arround_points = pycassa.ColumnFamily(pool2, "fz_arround_points")