Example #1
0
    def open(self):
        server_list = [self.host + ":" + self.port]
        connected = False

        try:
            if self.username and self.password:
                credentials = {
                    'username': self.username,
                    'password': self.password
                }
                self.pool = ConnectionPool(keyspace=self.keyspace,
                                           server_list=server_list,
                                           credentials=credentials,
                                           pool_size=self.pool_size)
                if self.pool:
                    self.logged_in = True
                    connected = True
            else:
                self.pool = ConnectionPool(keyspace=self.keyspace,
                                           server_list=server_list,
                                           pool_size=self.pool_size)
                if self.pool:
                    connected = True

        except Exception, e:
            pass
    def __init__(self, namespace, server_list=['localhost:9160']):
        # save cassandra server
        self.server_list = server_list
        self.namespace = namespace
        self._closed = False

        #setup_logging(self)

        # Connect to the server creating the namespace if it doesn't
        # already exist
        try:
            self.pool = ConnectionPool(namespace,
                                       self.server_list,
                                       max_retries=500,
                                       pool_timeout=600,
                                       timeout=10)
        except pycassa.InvalidRequestException:
            self._create_namespace(namespace)
            self.pool = ConnectionPool(namespace,
                                       self.server_list,
                                       max_retries=500,
                                       pool_timeout=600,
                                       timeout=10)

        try:
            self._tasks = pycassa.ColumnFamily(self.pool, 'tasks')
        except pycassa.NotFoundException:
            self._create_column_family('tasks',
                                       key_validation_class=ASCII_TYPE,
                                       bytes_columns=['task_data'])
            self._tasks = pycassa.ColumnFamily(self.pool, 'tasks')

        try:
            self._available = pycassa.ColumnFamily(self.pool, 'available')
        except pycassa.NotFoundException:
            self._create_column_family('available',
                                       key_validation_class=ASCII_TYPE,
                                       bytes_columns=['available'])
            self._available = pycassa.ColumnFamily(self.pool, 'available')

        try:
            self._task_count = pycassa.ColumnFamily(self.pool, 'task_count')
        except pycassa.NotFoundException:
            self._create_counter_column_family('task_count',
                                               key_validation_class=ASCII_TYPE,
                                               counter_columns=['task_count'])
            self._task_count = pycassa.ColumnFamily(self.pool, 'task_count')
            self._task_count.insert('RowKey', {'task_count': 0})

        try:
            self._available_count = pycassa.ColumnFamily(
                self.pool, 'available_count')
        except pycassa.NotFoundException:
            self._create_counter_column_family(
                'available_count',
                key_validation_class=ASCII_TYPE,
                counter_columns=['available_count'])
            self._available_count = pycassa.ColumnFamily(
                self.pool, 'available_count')
            self._available_count.insert('RowKey', {'available_count': 0})
Example #3
0
    def __init__(self, client, qname):
        super(Processor, self).__init__(client, qname)
        self.pool = ConnectionPool('processing_llama_Processor')
        self.trends = ColumnFamily(self.pool, 'Trend')

	def get_sleep_time():
		return 60
Example #4
0
 def _update_analytics_start_time(self, start_time):
     if mockcassandra.use_cql():
         cluster = Cluster(['127.0.0.1'],
                           port=int(self.__class__.cassandra_port))
         session = cluster.connect(COLLECTOR_KEYSPACE_CQL)
         query = "INSERT INTO {0} (key, \"{1}\") VALUES ('{2}', {3})".format(
             SYSTEM_OBJECT_TABLE, SYSTEM_OBJECT_START_TIME,
             SYSTEM_OBJECT_ANALYTICS, start_time)
         try:
             session.execute(query)
         except Exception as e:
             logging.error("INSERT INTO %s: Key %s Column %s Value %d "
                           "FAILED: %s" %
                           (SYSTEM_OBJECT_TABLE, SYSTEM_OBJECT_ANALYTICS,
                            SYSTEM_OBJECT_START_TIME, start_time, str(e)))
             assert False
         else:
             cluster.shutdown()
     else:
         pool = ConnectionPool(
             COLLECTOR_KEYSPACE,
             ['127.0.0.1:%s' % (self.__class__.cassandra_port)])
         col_family = ColumnFamily(pool, SYSTEM_OBJECT_TABLE)
         col_family.insert(SYSTEM_OBJECT_ANALYTICS,
                           {SYSTEM_OBJECT_START_TIME: start_time})
 def test_caching_pre_11(self):
     version = tuple(
         [int(v) for v in sys._conn.describe_version().split('.')])
     if version >= (19, 30, 0):
         raise SkipTest('CF specific caching no longer supported.')
     sys.create_column_family(TEST_KS,
                              'CachedCF10',
                              row_cache_size=100,
                              key_cache_size=100,
                              row_cache_save_period_in_seconds=3,
                              key_cache_save_period_in_seconds=3)
     pool = ConnectionPool(TEST_KS)
     cf = ColumnFamily(pool, 'CachedCF10')
     assert_equal(cf._cfdef.row_cache_size, 100)
     assert_equal(cf._cfdef.key_cache_size, 100)
     assert_equal(cf._cfdef.row_cache_save_period_in_seconds, 3)
     assert_equal(cf._cfdef.key_cache_save_period_in_seconds, 3)
     sys.alter_column_family(TEST_KS,
                             'CachedCF10',
                             row_cache_size=200,
                             key_cache_size=200,
                             row_cache_save_period_in_seconds=4,
                             key_cache_save_period_in_seconds=4)
     cf1 = ColumnFamily(pool, 'CachedCF10')
     assert_equal(cf1._cfdef.row_cache_size, 200)
     assert_equal(cf1._cfdef.key_cache_size, 200)
     assert_equal(cf1._cfdef.row_cache_save_period_in_seconds, 4)
     assert_equal(cf1._cfdef.key_cache_save_period_in_seconds, 4)
Example #6
0
def get_cassandra_connection(keyspace_name, hosts):
    key = keyspace_name, tuple(hosts)
    connection_pool, created_at = connection_pool_cache.get(key, (None, None))

    init_new_pool = connection_pool is None or connection_pool_expired(
        created_at)

    if connection_pool is not None and len(connection_pool.server_list) == 0:
        logging.error('connection pool had no active hosts')
        init_new_pool = True

    if init_new_pool:
        nodes = detect_nodes(hosts, keyspace_name)
        logger.info('setting up a new connection pool')
        connection_pool = ConnectionPool(
            keyspace_name,
            nodes,
            pool_size=settings.FEEDLY_CASSANDRA_CONNECTION_POOL_SIZE,
            prefill=False,
            timeout=settings.FEEDLY_CASSANDRA_TIMEOUT,
            max_retries=3)
        listener = FeedlyPoolListener(connection_pool)
        connection_pool.add_listener(listener)
        connection_pool_cache[key] = (connection_pool, time.time())
    return connection_pool
Example #7
0
def connect_server(address, key_space, column_family):
    """Establish connection."""
    pool = ConnectionPool(key_space, address)
    print "[INFO] Connection to '" + key_space + "' established."
    cf = ColumnFamily(pool, column_family)
    print "[INFO] Column family '" + column_family + "' used."
    return pool, cf
Example #8
0
    def __init__(self, client, qname, trend=5):
        super(Buyer, self).__init__(client, uuid.uuid4().hex)
        self.holdings = {}
        self.cash = 100000.0
        self.history = {}
        self.trend = trend
        self.pool = ConnectionPool('example_consumer_Buyer')
        self.stored_holdings = ColumnFamily(self.pool, 'Holdings')
        self.quote_history = ColumnFamily(self.pool, 'Quotes')
        self.stored_cash = ColumnFamily(self.pool, 'Cash')

        try:
            cash = self.stored_cash.get('current')
            self.cash = cash['amount']
        except ttypes.NotFoundException:
            self.stored_cash.insert('current', {'amount': self.cash})

        for symbol, columns in self.stored_holdings.get_range():
            self.holdings[symbol] = (columns['number_of_shares'],
                                     columns['price'], columns['cost'])

        date_expression = create_index_expression('timestamp',
                                                  datetime.date.today(), GT)
        date_clause = create_index_clause([date_expression], count=1000)

        for key, columns in self.quote_history.get_range():
            symbol = columns['symbol']
            price = columns['price']
            self.add_quote(symbol, price)
Example #9
0
 def _update_analytics_start_time(self, start_time):
     pool = ConnectionPool(
         COLLECTOR_KEYSPACE,
         ['127.0.0.1:%s' % (self.__class__.cassandra_port)])
     col_family = ColumnFamily(pool, SYSTEM_OBJECT_TABLE)
     col_family.insert(SYSTEM_OBJECT_ANALYTICS,
                       {SYSTEM_OBJECT_START_TIME: start_time})
def main():
    default_db_host = socket.gethostname()
    default_db_port = "9160"
    default_key_space = "ContrailAnalytics"
    default_table_space = "FlowRecordTable"

    try:
        (opts, args) = getopt.getopt(sys.argv[1:], "n:p:k:t:sh", \
          ["summary", "help"])
    except getopt.GetoptError:
        usage()

    show_summary = False
    for o, a in opts:
        #    (k, v) = opt
        if o in ["-h", "--help"]:
            usage()
        elif o in ["-n", "--host"]:
            default_db_host = a
        elif o in ["-p", "--port"]:
            default_db_port = a
        elif o in ["-k", "--key"]:
            default_key_space = a
        elif o in ["-t", "--table"]:
            default_table_space = a
        elif o in ["-s", "--summary"]:
            show_summary = True

    try:
        level = 1
        pool = ConnectionPool(
            keyspace=default_key_space,
            server_list=[default_db_host + ":" + default_db_port],
            timeout=0.5,
            max_retries=2)
        f = pycassa.ColumnFamily(pool, default_table_space)
        row_count = sum(1 for _ in f.get_range())
        if (row_count >= 0):
            print "HTTP/1.1 200 OK"
            print "Content-Type: Content-Type: text/plain"
            print
            print "ColumnFamily: %s in KeySpace:%s has %d rows" % (
                default_table_space, default_key_space, row_count)
            print
            print "Cassandra checked!"
            sys.exit(0)
        else:
            print "HTTP/1.1 500 Internal Server Error"
            print "Content-Type: Content-Type: text/plain"
            print
            print "Unable to query ColumnFamily %s" % (default_table_space)
            sys.exit(1)
    except Exception as e:
        print "HTTP/1.1 503 Service Unavailable"
        print "Content-Type: Content-Type: text/plain"
        print
        print str(e)
        sys.exit(2)
Example #11
0
 def setUp(self):
     n = 10000
     self.weibo_ids = self._load_items(n)
     pool = ConnectionPool(
         'master_timeline',
         server_list=['219.224.135.60:9160', '219.224.135.61:9160'],
         pool_size=10)
     col_fam = pycassa.ColumnFamily(pool, 'weibos')
     self.weibos_col_fam = col_fam
Example #12
0
    def do_use(self, keyspace):
        if keyspace not in self.sm.list_keyspaces():
            return self.perror('Unknown keyspace %s' % keyspace)

        self.prompt = '%s/%s> ' % (self.server, keyspace)
        self.keyspace = keyspace
        self.pool = ConnectionPool(keyspace, server_list=[self.server])
        
        print 'Using %s as default keyspace' % keyspace
Example #13
0
 def __init__(self, keyspace=None, host='localhost', port='9160'):
     self.host = host
     self.port = port
     self.keyspace = keyspace
     self.pool = ConnectionPool(self.keyspace,
                                [self.host + ':' + self.port])
     self.providerCF = pycassa.ColumnFamily(self.pool, self.providerCFname)
     self.patientCF = pycassa.ColumnFamily(self.pool, self.patientCFname)
     self.patientMapCF = pycassa.ColumnFamily(self.pool,
                                              self.patientMapCFname)
Example #14
0
 def __init__(self, keyspacename, serverport):
     self.serverport = serverport
     self.keyspace = keyspacename 
     self.sys = SystemManager(self.serverport)   
     self.cfs = self.sys.get_keyspace_column_families(self.keyspace)
     self.keyspacenames = self.cfs.keys()
     self.pool = ConnectionPool(self.keyspace, [ self.serverport ])
     self.column_family = {}
     self.createtables()
     self.reinit_column_family_map()
    def verify_with_thrift(self):
        # No more thrift in 4.0
        if self.cluster.version() >= '4':
            return

        pool = ConnectionPool("supcols", pool_size=1)
        super_col_fam = ColumnFamily(pool, "cols")
        for name in NAMES:
            super_col_value = super_col_fam.get(name)
            self.assertEqual(OrderedDict([(('attr', u'name'), name)]),
                             super_col_value)
Example #16
0
 def connect_db(self):
     try:
         self._pool = ConnectionPool(
             COLLECTOR_KEYSPACE,
             server_list=self._cassandra_server_list,
             timeout=None)
     except Exception as e:
         self._logger.error("Exception: Failure in connection to "
                            "AnalyticsDb %s" % e)
         return -1
     return None
    def setUp(self):
        self.sysman = SystemManager()
        self.sysman.create_keyspace(
            TEST_KS, SIMPLE_STRATEGY,
            dict(replication_factor='1')
        )
        self.sysman.create_column_family(TEST_KS, TEST_CF)

        self.pool = ConnectionPool(TEST_KS)

        self.cf = ColumnFamily(self.pool, TEST_CF)
Example #18
0
def feeder(start_point_dict):
    frontend = {}
    backend = {}
    try:
        frontend['pool'] = ConnectionPool(F_KS, F_SERVLIST)
        frontend['handle'] = ColumnFamily(frontend['pool'], F_CF)
        backend['pool'] = ConnectionPool(B_KS, B_SERVLIST)
        backend['handle'] = ColumnFamily(backend['pool'], B_CF)
    except pycassa.NotFoundException as err:
        print "[ERROR] " + str(err)
        exit(-1)
    except pycassa.AllServersUnavailable as err:
        print "[ERROR] " + str(err)
        exit(-1)

    counter = 0
    for sp in start_point_dict:
        filelist = get_filelist(start_point_dict[sp])
        if not filelist:
            print "[WARN] No files were found."
            continue

        for filename in filelist:
            if os.path.getsize(filename) > SIZE_LIMIT:
                continue
            file_content = ""
            with open(filename, "rb") as f:
                file_content = f.read()

            hasherlist = [hashlib.md5(), hashlib.sha1()]
            taskid = gen_taskid(hasherlist, filename, file_content)
            print "%s\n%s\n%s" % (sp, filename, taskid)
            frontend['handle'].insert(filename, {sp: taskid})
            backend['handle'].insert(taskid, {'content': file_content})
            print "Uploaded\n"
            counter += 1

    print "Total " + str(counter) + " file(s) uploaded."

    frontend['pool'].dispose()
    backend['pool'].dispose()
Example #19
0
def verify_cassandra(thriftport, cqlport, cassandra_user, cassandra_password):
    retry_threshold = 10
    retry = 1
    while retry < retry_threshold:
        try:
            if cassandra_user is None and cassandra_password is None:
                pool = ConnectionPool(None, ['localhost:' + str(thriftport)])
            else:
                cred = {
                    'username': cassandra_user,
                    'password': cassandra_password
                }
                pool = ConnectionPool(None, ['localhost:'+str(thriftport)], \
                                      credentials=cred)
            return True
        except Exception as e:
            logging.info("Exception: Failure in connection to "
                         "AnalyticsDb: Retry %d: %s" % (retry, e))
            retry = retry + 1
            time.sleep(5)
    return False
    def test_alter_column_non_bytes_type(self):
        sys.create_column_family(TEST_KS, 'LongCF', comparator_type=LONG_TYPE)
        sys.create_index(TEST_KS, 'LongCF', 3, LONG_TYPE)
        pool = ConnectionPool(TEST_KS)
        cf = ColumnFamily(pool, 'LongCF')
        cf.insert('key', {3: 3})
        assert_equal(cf.get('key')[3], 3)

        sys.alter_column(TEST_KS, 'LongCF', 2, LONG_TYPE)
        cf = ColumnFamily(pool, 'LongCF')
        cf.insert('key', {2: 2})
        assert_equal(cf.get('key')[2], 2)
    def test_alter_column_family_default_validation_class(self):
        sys.create_column_family(TEST_KS,
                                 'AlteredCF',
                                 default_validation_class=LONG_TYPE)
        pool = ConnectionPool(TEST_KS)
        cf = ColumnFamily(pool, 'AlteredCF')
        assert_equal(cf.default_validation_class, "LongType")

        sys.alter_column_family(TEST_KS,
                                'AlteredCF',
                                default_validation_class=UTF8_TYPE)
        cf = ColumnFamily(pool, 'AlteredCF')
        assert_equal(cf.default_validation_class, "UTF8Type")
 def column_family_remove(self, machine_id, keyspace_name,
                          column_family_name, key):
     """Remove a key from column family for a given keyspace """
     if (self.keyspace_contains(keyspace_name,
                                column_family_name) == False):
         print "Error : Keyspace:column family could not be found."
         return False
     pool = ConnectionPool(keyspace=keyspace_name,
                           server_list=keyspace.server_ips,
                           prefill=False)
     col_fam = ColumnFamily(pool, column_family_name)
     col_fam.remove(key)
     return True
Example #23
0
def atualiza_code():
    import sys
    sys.path.append('./pymongo')
    sys.path.append('./pycassa')
    import pymongo
    import pycassa
    from pycassa.pool import ConnectionPool
    from pycassa import index
    from pycassa.columnfamily import ColumnFamily
    #
    pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'],
                           timeout=10000000000)
    #
    tb_object1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_OBJECT')
    tb_object_dt1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_OBJECT_DT')
    tb_relaction1 = pycassa.ColumnFamily(pool2, 'SEMANTIC_RELACTIONS')
    tb_know1 = pycassa.ColumnFamily(pool2, 'knowledge_manager')
    #
    MONGO_URL = 'mongodb://*****:*****@ds061938.mongolab.com:61938/mdnet'
    conn = pymongo.Connection(MONGO_URL)
    dbM = conn.mdnet
    #
    tb_object = dbM['SEMANTIC_OBJECT']
    tb_object_dt = dbM['SEMANTIC_OBJECT_DT']
    tb_relaction = dbM['SEMANTIC_RELACTIONS']
    tb_know = dbM['knowledge_manager']
    #
    tb_object.remove()
    tb_object_dt.remove()
    tb_relaction.remove()
    tb_know.remove()
    #
    r1 = tb_object1.get_range()
    for k, r in r1:
        r['id'] = k
        tb_object.insert(r)
    #
    r1 = tb_object_dt1.get_range()
    for k, r in r1:
        r['id'] = k
        tb_object_dt.insert(r)

    r1 = tb_relaction1.get_range()
    for k, r in r1:
        r['id'] = k
        tb_relaction.insert(r)
    #===
    r1 = tb_know1.get_range()
    for k, r in r1:
        r['id'] = k
        tb_know.insert(r)
Example #24
0
 def setUp(self):
     self.keyspace = 'Claimspace'
     self.host = 'localhost'
     self.port = '9160'
     self.pool = ConnectionPool(self.keyspace,
                                [self.host + ':' + self.port])
     self.cf = pycassa.ColumnFamily(self.pool, Database.providerCFname)
     self.db = Database(self.keyspace)
     self.providerDict = {
         'username': '******',
         'realname': 'Testy User',
         'providerid': '8675309'
     }
     self.providerObj = Provider(**self.providerDict)
Example #25
0
 def reinit_column_family_map(self):
     self.cfs = self.sys.get_keyspace_column_families(self.keyspace)
     self.keyspacenames = self.cfs.keys()
     self.pool = ConnectionPool(self.keyspace, [ self.serverport ] )
     self.column_family = {}
     for ks in self.keyspacenames:
         self.column_family[ks] = ColumnFamily(self.pool, ks)
     self.column_family['SimpleTable']._set_key_validation_class(IntegerType())
     self.column_family['Threads']._set_key_validation_class(IntegerType())
     self.column_family['UserThreads']._set_key_validation_class(IntegerType())
     self.column_family['ThreadObjects']._set_key_validation_class(IntegerType())
     self.column_family['Objects']._set_key_validation_class(UUIDType())
     self.column_family['ThreadHolds']._set_key_validation_class(IntegerType())
     self.column_family['ObjectHistory']._set_key_validation_class(UUIDType())
 def colum_family_insert(self, machine_id, keyspace_name,
                         column_family_name, user_content):
     """Insert into a column family for a given keyspace """
     if (self.keyspace_contains(keyspace_name,
                                column_family_name) == False):
         print "Error : Keyspace:column family could not be found."
         return False
     pool = ConnectionPool(keyspace=keyspace_name,
                           server_list=keyspace.server_ips,
                           prefill=False)
     col_fam = ColumnFamily(pool, column_family_name)
     for content in user_content:
         col_fam.insert(content, user_content[content])
     #col_fam.insert('Key2', {'name':'mayur', 'age':'23'})
     return True
 def start_socket():
     global sock
     global pool2
     while True:
         try:
             pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'],
                                    timeout=10000000000)
             mdTb.start_db(pool2)
             get_object.start_db(pool2)
             get_object2.start_db(pool2)
             Identify.start_db(pool2)
         except Exception, e:
             print 'Not connected(cassandra)....', e
             time.sleep(5)
             continue
         break
    def test_column_validators(self):
        validators = {'name': UTF8_TYPE, 'age': LONG_TYPE}
        sys.create_column_family(TEST_KS,
                                 'ValidatedCF',
                                 column_validation_classes=validators)
        pool = ConnectionPool(TEST_KS)
        cf = ColumnFamily(pool, 'ValidatedCF')
        cf.insert('key', {'name': 'John', 'age': 40})
        self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})

        validators = {'name': ASCII_TYPE, 'age': INT_TYPE}
        sys.alter_column_family(TEST_KS,
                                'ValidatedCF',
                                column_validation_classes=validators)
        cf.load_schema()
        self.assertEquals(cf.get('key'), {'name': 'John', 'age': 40})
Example #29
0
def get_values(servlst, ks, cf, key):
    #print key
    try:
        pool = ConnectionPool(ks, servlst)
        cf_handle = ColumnFamily(pool, cf)
        result = cf_handle.get(key).values()
    except pycassa.NotFoundException as err:
        print "[ERROR] " + key + " not found"
        result = ""
    except Exception as err:
        print "[ERROR] " + str(err)
        exit(-1)
    finally:
        pool.dispose()

    return result
Example #30
0
def init_base():
    global pool2
    global tb_fuzzy
    global tb_fz_store_pref
    global tb_fz_store_defs
    global tb_fz_store_sufix
    global tb_fz_store_refer
    global tb_fz_arround_points
    #
    pool2 = ConnectionPool('MINDNET', ['79.143.185.3:9160'], timeout=10000)
    tb_fuzzy = pycassa.ColumnFamily(pool2, 'fuzzy_store')
    tb_fz_store_pref = pycassa.ColumnFamily(pool2, "fz_store_pref")
    tb_fz_store_defs = pycassa.ColumnFamily(pool2, "fz_store_defs")
    tb_fz_store_sufix = pycassa.ColumnFamily(pool2, "fz_store_sufix")
    tb_fz_store_refer = pycassa.ColumnFamily(pool2, "fz_store_refer")
    tb_fz_arround_points = pycassa.ColumnFamily(pool2, "fz_arround_points")