Exemple #1
0
class Cluster(object):
    def __init__(self, name):
        self.keyspaces = OrderedDict()
        self.name = name
        self._sm_client = None
        
        cmcache.append('clusters', self)
    
    def setclient(self, servers_list=[connection.DEFAULT_SERVER]):
        for server in servers_list:
            try:
                self._sm_client = SystemManager(server)
                return
            except:
                pass
        if self._sm_client is None:
            raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list))
    
    def registerKeyspace(self, name, keyspc):
        self.keyspaces[name] = keyspc
        if self._sm_client is None:
            self.setclient()
        # try to actually register this keyspace to the system via 
        if not name in self._sm_client.list_keyspaces():
            self._sm_client.create_keyspace(name, keyspec.strategy_class)
        
    def __str__(self):
        return self.name
Exemple #2
0
 def tearDown(self):
     if self.store._store is not None:
         self.store.clear()
         self.store.close()
     from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
     system_manager = SystemManager('localhost:9160')
     system_manager.drop_column_family('Murk', 'shove')
Exemple #3
0
 def tearDown(self):
     if self.store._store is not None:
         self.store.clear()
         self.store.close()
     from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
     system_manager = SystemManager('localhost:9160')
     system_manager.drop_column_family('Murk', 'shove')
Exemple #4
0
    def db_export(self):
        db_contents = {'cassandra': {}, 'zookeeper': {}}

        cassandra_contents = db_contents['cassandra']
        for ks_name in (set(KEYSPACES) - set(self._args.omit_keyspaces or [])):
            if self._api_args.cluster_id:
                full_ks_name = '%s_%s' % (self._api_args.cluster_id, ks_name)
            else:
                full_ks_name = ks_name
            cassandra_contents[ks_name] = {}

            socket_factory = pycassa.connection.default_socket_factory
            if self._api_args.cassandra_use_ssl:
                socket_factory = pycassa.connection.make_ssl_socket_factory(
                    self._api_args.cassandra_ca_certs, validate=False)
            pool = pycassa.ConnectionPool(full_ks_name,
                                          self._api_args.cassandra_server_list,
                                          pool_timeout=120,
                                          max_retries=-1,
                                          timeout=5,
                                          socket_factory=socket_factory)

            creds = None
            if (self._api_args.cassandra_user
                    and self._api_args.cassandra_password):
                creds = {
                    'username': self._api_args.cassandra_user,
                    'password': self._api_args.cassandra_password
                }
            sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
                                    credentials=creds)
            for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
                cassandra_contents[ks_name][cf_name] = {}
                cf = pycassa.ColumnFamily(pool, cf_name)
                for r, c in cf.get_range(column_count=10000000,
                                         include_timestamp=True):
                    cassandra_contents[ks_name][cf_name][r] = c

        def get_nodes(path):
            if not zk.get_children(path):
                return [(path, zk.get(path))]

            nodes = []
            for child in zk.get_children(path):
                nodes.extend(get_nodes('%s%s/' % (path, child)))

            return nodes

        zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
        zk.start()
        nodes = get_nodes(self._api_args.cluster_id + '/')
        zk.stop()
        db_contents['zookeeper'] = json.dumps(nodes)

        f = open(self._args.export_to, 'w')
        try:
            f.write(json.dumps(db_contents))
        finally:
            f.close()
 def setup_class(cls):
     sys = SystemManager()
     sys.create_column_family(
         TEST_KS,
         'CustomComposite1',
         comparator_type=CompositeType(
             IntegerType(),
             UTF8Type()))
    def test_single_component_composite(self):
        sys = SystemManager()
        sys.create_column_family(TEST_KS, 'SingleComposite',
                comparator_type=CompositeType(IntegerType()))

        cf = ColumnFamily(pool, 'SingleComposite')
        cf.insert('key', {(123456,): 'val'})
        assert_equal(cf.get('key'), {(123456,): 'val'})
Exemple #7
0
 def setUp(self):
     from shove import Shove
     from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
     system_manager = SystemManager('localhost:9160')
     try:
         system_manager.create_column_family('Murk', 'shove')
     except:
         pass
     self.store = Shove('cassandra://localhost:9160/Murk/shove')
 def setUp(self):
     from shove import Shove
     from pycassa.system_manager import SystemManager
     system_manager = SystemManager('localhost:9160')
     try:
         system_manager.create_column_family('Foo', 'shove')
     except:
         pass
     self.store = Shove('cassandra://localhost:9160/Foo/shove')
 def setUp(self):
     from shove import Shove
     from pycassa.system_manager import SystemManager
     system_manager = SystemManager('localhost:9160')
     try:
         system_manager.create_column_family('Foo', 'shove')
     except:
         pass
     self.store = Shove('cassandra://localhost:9160/Foo/shove')
Exemple #10
0
 def setUp(self):
     from shove import Shove
     from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
     system_manager = SystemManager('localhost:9160')
     try:
         system_manager.create_column_family('Murk', 'shove')
     except:
         pass
     self.store = Shove('cassandra://localhost:9160/Murk/shove')
Exemple #11
0
 def delete_namespace(self):
     sm = SystemManager(self._chosen_server)
     try:
         sm.drop_keyspace(self._app_namespace)
     except pycassa.InvalidRequestException, exc:
         if exc.why.startswith('Cannot drop non existing keyspace'):
             pass
         else:
             raise exc
    def db_export(self):
        db_contents = {'cassandra': {},
                       'zookeeper': {}}

        cassandra_contents = db_contents['cassandra']
        for ks_name in (set(KEYSPACES) -
                        set(self._args.omit_keyspaces or [])):
            if self._api_args.cluster_id:
                full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
            else:
                full_ks_name = ks_name
            cassandra_contents[ks_name] = {}

            socket_factory = pycassa.connection.default_socket_factory
            if self._api_args.cassandra_use_ssl:
                socket_factory = pycassa.connection.make_ssl_socket_factory(
                    self._api_args.cassandra_ca_certs, validate=False)
            pool = pycassa.ConnectionPool(
                full_ks_name, self._api_args.cassandra_server_list,
                pool_timeout=120, max_retries=-1, timeout=5,
                socket_factory=socket_factory)

            creds = None
            if (self._api_args.cassandra_user and
                self._api_args.cassandra_password):
                creds = {'username': self._api_args.cassandra_user,
                         'password': self._api_args.cassandra_password}
            sys_mgr = SystemManager(self._api_args.cassandra_server_list[0],
                credentials=creds)
            for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
                cassandra_contents[ks_name][cf_name] = {}
                cf = pycassa.ColumnFamily(pool, cf_name)
                for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
                    cassandra_contents[ks_name][cf_name][r] = c

        def get_nodes(path):
            if not zk.get_children(path):
                return [(path, zk.get(path))]

            nodes = []
            for child in zk.get_children(path):
                nodes.extend(get_nodes('%s%s/' %(path, child)))

            return nodes

        zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
        zk.start()
        nodes = get_nodes(self._api_args.cluster_id+'/')
        zk.stop()
        db_contents['zookeeper'] = json.dumps(nodes)

        f = open(self._args.export_to, 'w')
        try:
            f.write(json.dumps(db_contents))
        finally:
            f.close()
Exemple #13
0
 def setup_class(cls):
     sys = SystemManager()
     sys.create_column_family(TEST_KS, 'StaticComposite',
                              comparator_type=CompositeType(LongType(),
                                                            IntegerType(),
                                                            TimeUUIDType(reversed=True),
                                                            LexicalUUIDType(reversed=False),
                                                            AsciiType(),
                                                            UTF8Type(),
                                                            BytesType()))
def CreateCF(connStr, keySpace, cfs, columesTypeList = None):
    sysmgt = SystemManager(connStr)
    list = sysmgt.list_keyspaces()
    
    if keySpace in list:
        #print('pyagent is existed')
        pass
    else:
        sysmgt.create_keyspace(keySpace, SIMPLE_STRATEGY, {'replication_factor': '3'})
    cfDict = sysmgt.get_keyspace_column_families(keySpace)
    allCF = cfDict.keys()
    
    for i in range(len(cfs)):
        columnFamily = cfs[i]
        #print(columnFamily)
        if columesTypeList != None:
            columes = columesTypeList[i]
        else:
            columes = None
        if columnFamily in allCF:
            #print(columnFamily + ' is existed')
            pass
        else:
            #print(columnFamily + ' is creating')
            sysmgt.create_column_family(keySpace, columnFamily, super=False, 
                comparator_type=UTF8_TYPE,
                key_validation_class=ASCII_TYPE,
                default_validation_class=UTF8_TYPE,
                column_validation_classes=columes,
                gc_grace_seconds=1000)
    sysmgt.close()
Exemple #15
0
def get_connection():
    """
    Creates a connection to Cassandra.

    Returs:
        pool
    """
    cassandra_host = os.environ.get('CASSANDRA_HOST', 'localhost')
    sys_mgr = SystemManager()
    try:
        sys_mgr.describe_ring(KEYSPACE)
    except:
        sys_mgr.create_keyspace(KEYSPACE, SIMPLE_STRATEGY, {'replication_factor': '1'})

    pool = ConnectionPool(KEYSPACE, server_list=[cassandra_host])
    for cf_name in [CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST, CF_LOGS_BY_SEVERITY]:
        try:
            cf = ColumnFamily(pool, cf_name)
        except:
            sys_mgr.create_column_family(KEYSPACE, cf_name, comparator_type=TimeUUIDType())
            cf = ColumnFamily(pool, cf_name)
            cf.get_count(str(uuid.uuid4()))

    sys_mgr.close()

    return pool
Exemple #16
0
def main(argv):
    usage = "Usage: dbdiscover.py"

    args, kwargs = parse(argv)

    host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
    port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)

    try:
        system = SystemManager("%s:%s" % (host, port))
        ksprops = system.get_keyspace_properties("system").keys()
        #        header = ["keyspace"] + sorted(ksprops)
        header = ["keyspace", "column_family"]
        writer = csv.writer(output)
        writer.writerow(header)

        for keyspace in system.list_keyspaces():
            ksinfo = system.get_keyspace_column_families(
                keyspace, use_dict_for_col_metadata=True)
            attrs = [
                "id",
                "column_type",
                "comment",
                "comparator_type",
                "default_validation_class",
                "gc_grace_seconds",
                "key_alias",
                "key_cache_save_period_in_seconds",
                "key_cache_size",
                "key_validation_class",
                "max_compaction_threshold",
                "memtable_operations_in_millions",
                "memtable_throughput_in_mb",
                "memtable_flush_after_mins",
                "merge_shards_chance",
                "min_compaction_threshold",
                "read_repair_chance",
                "replicate_on_write",
                "row_cache_provider",
                "row_cache_save_period_in_seconds",
                "row_cache_size",
                "subcomparator_type",
            ]

            for cfname, cfdef in ksinfo.iteritems():
                ks = ("keyspace=" + keyspace)
                cf = ("column_family=" + cfname)
                row = (ks, cf)
                writer.writerow(row)

        return

    except:
        error(output, excinfo(), 2)
    def setUp(self):
        self.sysman = SystemManager()
        self.sysman.create_keyspace(
            TEST_KS, SIMPLE_STRATEGY,
            dict(replication_factor='1')
        )
        self.sysman.create_column_family(TEST_KS, TEST_CF)

        self.pool = ConnectionPool(TEST_KS)

        self.cf = ColumnFamily(self.pool, TEST_CF)
def main(argv):
    usage = "Usage: dbdiscover.py"
    
    args, kwargs = parse(argv)

    host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
    port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)

    try:
        system = SystemManager("%s:%s" % (host, port))
        ksprops = system.get_keyspace_properties("system").keys()
#        header = ["keyspace"] + sorted(ksprops)
        header = ["keyspace", "column_family"]
        writer = csv.writer(output)
        writer.writerow(header)

        for keyspace in system.list_keyspaces():
            ksinfo = system.get_keyspace_column_families(keyspace, use_dict_for_col_metadata=True)
            attrs = [
            "id",
            "column_type",
            "comment",
            "comparator_type",
            "default_validation_class",
            "gc_grace_seconds",
            "key_alias",
            "key_cache_save_period_in_seconds",
            "key_cache_size",
            "key_validation_class",
            "max_compaction_threshold",
            "memtable_operations_in_millions",
            "memtable_throughput_in_mb",
            "memtable_flush_after_mins",
            "merge_shards_chance",
            "min_compaction_threshold",
            "read_repair_chance",
            "replicate_on_write",
            "row_cache_provider",
            "row_cache_save_period_in_seconds",
            "row_cache_size",
            "subcomparator_type",
            ]

            for cfname, cfdef in ksinfo.iteritems():
                ks = ("keyspace="+ keyspace)
                cf = ("column_family="+ cfname)
                row =(ks, cf)
                writer.writerow(row)


        return

    except:
        error(output, excinfo(), 2)
    def handle(self, **options):
        """

        :param options:
        """
        sys = SystemManager(server=CASSANDRA_HOSTS[0])
        sys.create_column_family(CASSANDRA_SESSIONS_KEYSPACE, CASSANDRA_SESSIONS_COLUMN_FAMILY,
                                 key_validation_class=types.UTF8Type(),
                                 column_validation_classes={
                                     'session_data': types.UTF8Type()
                                 })
        sys.close()
Exemple #20
0
def create():
    keyspace = configuration.cassandra_keyspace
    mgr = SystemManager()
    try:
        workaround_1779(mgr.create_column_family, keyspace, 'Indexes',
            comparator_type=UTF8_TYPE)
        workaround_1779(mgr.create_column_family, keyspace, 'Stacktrace',
            comparator_type=UTF8_TYPE)
        workaround_1779(mgr.create_column_family, keyspace, 'AwaitingRetrace',
            comparator_type=UTF8_TYPE)
    finally:
        mgr.close()
Exemple #21
0
 def _create_cf(self, cf_name):
     sys_mng = SystemManager('localhost:9160')
     validators = {'name': UTF8_TYPE, 'content': UTF8_TYPE, 'post_time': UTF8_TYPE}
     sys_mng.create_column_family(
         'bbs',
         cf_name,
         super=False, 
         comparator_type=UTF8_TYPE,
         key_validation_class=UTF8_TYPE, 
         column_validation_classes=validators
     )
     sys_mng.close()
Exemple #22
0
    def test_uuid_composites(self):
        sys = SystemManager()
        sys.create_column_family(TEST_KS, 'UUIDComposite',
                comparator_type=CompositeType(IntegerType(reversed=True), TimeUUIDType()),
                key_validation_class=TimeUUIDType(),
                default_validation_class=UTF8Type())

        key, u1, u2 = uuid.uuid1(), uuid.uuid1(), uuid.uuid1()
        cf = ColumnFamily(pool, 'UUIDComposite')
        cf.insert(key, {(123123, u1): 'foo'})
        cf.insert(key, {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
        assert_equal(cf.get(key), {(123123, u1): 'foo', (-1, u2): 'bar', (-123123123, u1): 'baz'})
Exemple #23
0
    def do_connect(self, server):
        try:
            self.sm = SystemManager(server)
        except TTransportException:
            return self.perror('Can not connect to %s' % server)
        
        self.server = server
        self.prompt = '%s> ' % server

        self.keyspace = None
        self.pool = None

        print 'Successfully connected to %s' % server
Exemple #24
0
 def wait_for_consistency(self, sm=None):
     if sm is None:
         sm = SystemManager(self._chosen_server)
     start_consistency_delay = time.time()
     consistency_delay = 0
     while len(sm.describe_schema_versions()) > 1 and \
             consistency_delay < self.max_consistency_delay:
         consistency_delay = time.time() - start_consistency_delay
         if consistency_delay > 20:
             logger.warn('waited %.1f seconds for cluster-wide consistency %r' % (
                     consistency_delay, sm.describe_schema_versions()))
         time.sleep(0.2)
     logger.info('number of schemas in cluster: %d' % len(sm.describe_schema_versions()))
def initializeTableLayout(keyspace, server_list, replicationStrategy,
  strategyOptions, localDCName, credentials):

    sys_manager = SystemManager(server_list[0], credentials=credentials)

    # Make sure the the keyspace exists
    if keyspace not in sys_manager.list_keyspaces():
      sys_manager.create_keyspace(keyspace, replicationStrategy,
        strategyOptions)

    cf_defs = sys_manager.get_keyspace_column_families(keyspace)

    # Create UTF8 CF's
    for tablename in ["global_nodes", "metadata"]:
      if tablename not in cf_defs.keys():
        createUTF8ColumnFamily(sys_manager, keyspace, tablename)

    if localDCName:

        dcNodes = "dc_%s_nodes" % (localDCName,)
        if dcNodes not in cf_defs.keys():
          createUTF8ColumnFamily(sys_manager, keyspace, dcNodes)
    else:
      # TODO Log we do not have the DC name
      pass

    if "node_slices" not in cf_defs.keys():
      sys_manager.create_column_family(
        keyspace,
        "node_slices",
        super=False,
        comparator_type=pycassa_types.LongType(),
        key_validation_class=pycassa_types.UTF8Type(),
        default_validation_class=pycassa_types.LongType()
      )
Exemple #26
0
 def test_get_indexed_slices(self):
     sys = SystemManager()
     for cf, keys in self.type_groups:
         sys.create_index(TEST_KS, cf.column_family, 'birthdate', LongType())
         cf = ColumnFamily(pool, cf.column_family)
         for key in keys:
             cf.insert(key, {'birthdate': 1})
         expr = create_index_expression('birthdate', 1)
         clause = create_index_clause([expr])
         rows = list(cf.get_indexed_slices(clause))
         assert_equal(len(rows), len(keys))
         for k, c in rows:
             assert_true(k in keys)
             assert_equal(c, {'birthdate': 1})
Exemple #27
0
    def test_add_remove_counter(self):
        sys = SystemManager()
        sys.create_column_family(TEST_KS, 'KeyLongCounter', key_validation_class=LongType(),
                                 default_validation_class='CounterColumnType')
        sys.close()
        cf_long = ColumnFamily(pool, 'KeyLongCounter')

        key = 1111111111111111L

        cf_long.add(key, 'col')
        assert_equal(cf_long.get(key), {'col': 1})
        cf_long.remove_counter(key, 'col')
        time.sleep(0.1)
        assert_raises(NotFoundException, cf_long.get, key)
Exemple #28
0
    def handle_noargs(self, **options):
        sys = SystemManager(server=settings.CASSANDRA_SERVERS[0])
        REPLICATION_STRATEGY = getattr(pycassa.system_manager, settings.CASSANDRA_REPLICATION_STRATEGY)

        existing_cfs = sys.get_keyspace_column_families(settings.CASSANDRA_KEYSPACE).keys()

        if 'APIConsumers' not in existing_cfs:
            print 'Creating missing column family: APIConsumers'
            sys.create_column_family(settings.CASSANDRA_KEYSPACE, 'APIConsumers', comparator_type=UTF8_TYPE)
            sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_key', UTF8_TYPE)
            sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'consumer_secret', UTF8_TYPE)
            sys.alter_column(settings.CASSANDRA_KEYSPACE, 'APIConsumers', 'username', UTF8_TYPE)

        print 'All done!'
 def __init__(self, engine, **kw):
     super(CassandraStore, self).__init__(engine, **kw)
     spliturl = urlsplit(engine)
     _, keyspace, column_family = spliturl.path.split('/')
     try:
         self._store = pycassa.ColumnFamily(
             pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
             column_family,
         )
     except pycassa.InvalidRequestException:
         from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
         system_manager = SystemManager(spliturl[1])
         system_manager.create_keyspace(
             keyspace,
             pycassa.system_manager.SIMPLE_STRATEGY,
             dict(replication_factor=native(kw.get('replication', 1))),
         )
         system_manager.create_column_family(keyspace, column_family)
         self._store = pycassa.ColumnFamily(
             pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
             column_family,
         )
     except pycassa.NotFoundException:
         from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
         system_manager = SystemManager(spliturl[1])
         system_manager.create_column_family(keyspace, column_family)
         self._store = pycassa.ColumnFamily(
             pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
             column_family,
         )
 def __init__(self, engine, **kw):
     super(CassandraStore, self).__init__(engine, **kw)
     spliturl = urlparse.urlsplit(engine)
     _, keyspace, column_family = spliturl[2].split('/')
     try:
         self._pool = pycassa.connect(keyspace, [spliturl[1]])
         self._store = pycassa.ColumnFamily(self._pool, column_family)
     except pycassa.InvalidRequestException:
         from pycassa.system_manager import SystemManager
         system_manager = SystemManager(spliturl[1])
         system_manager.create_keyspace(
             keyspace, kw.get('replication', 1)
         )
         system_manager.create_column_family(keyspace, column_family)
         self._pool = pycassa.connect(keyspace, [spliturl[1]])
         self._store = pycassa.ColumnFamily(self._pool, column_family)
Exemple #31
0
def main(argv):
    usage = "Usage: dbschema [options] [{resource}]"

    args, kwargs = parse(argv)

    host = kwargs.get('host', settings.DEFAULT_CASSANDRA_HOST)
    port = kwargs.get('port', settings.DEFAULT_CASSANDRA_PORT)

    try:
        system = SystemManager("%s:%s" % (host, port))

        #  FOR TESTING ONLY
        #        print str(argv)

        if len(argv) == 0:
            list_keyspaces(system)
            return

        if len(argv) == 1:
            ksname = argv[0]
            list_keyspace(system, ksname)

        if len(argv) == 2:
            ksname = argv[0]
            cfname = argv[1]
            if cfname != 'ANY':
                list_column_family(system, ksname, cfname)
            else:
                list_keyspace(system, ksname)

    except:
        error(output, excinfo(), 2)
Exemple #32
0
 def __init__(self):
     try:
         logger.info("Connection to " + DB_IP + ":" + DB_PORT + " ...")
         self.sysDB = SystemManager(DB_IP + ":" + DB_PORT)
         logger.info("Connection to the systemManager established.")
     except Exception as exep:
         logger.warning("Could not connect to the DB '" + DB_IP + ":" + DB_PORT + "'. | " + str(exep))
Exemple #33
0
def _sys_mgr():
    """
    Return a pycassa system manager connection object
    """
    thrift_port = str(__salt__["config.option"]("cassandra.THRIFT_PORT"))
    host = __salt__["config.option"]("cassandra.host")
    return SystemManager("{}:{}".format(host, thrift_port))
Exemple #34
0
def _sys_mgr():
    '''
    Return a pycassa system manager connection object
    '''
    thrift_port = str(__salt__['config.option']('cassandra.THRIFT_PORT'))
    host = __salt__['config.option']('cassandra.host')
    return SystemManager('{0}:{1}'.format(host, thrift_port))
 def ensure_cassandra_cf(self):
     s = SystemManager()
     if self.keyspace not in s.list_keyspaces():
         s.create_keyspace(self.keyspace, SIMPLE_STRATEGY, {'replication_factor': '1'})
     if self.cassandra_columns_family not in s.get_keyspace_column_families(self.keyspace):
         s.create_column_family(self.keyspace, self.cassandra_columns_family)
     self.columnfamily = ColumnFamily(self.cassandra_session, self.cassandra_columns_family)
Exemple #36
0
 def _create_tables(self, table_names, sm=None):
     if sm is None:
         sm = SystemManager(self._chosen_server)
     for family, num_uuids in table_names.items():
         #CompositeType(*[UUIDType() for i in range(num_uuids)])
         comparator = AsciiType()
         try:
             sm.create_column_family(
                 self._app_namespace, family, super=False,
                 key_validation_class = ASCII_TYPE,
                 default_validation_class = BYTES_TYPE,
                 comparator_type=comparator,
                 )
         except pycassa.InvalidRequestException, exc:
             if exc.why.startswith('Cannot add already existing column family'):
                 pass
             else:
                 raise exc
Exemple #37
0
 def setclient(self, servers_list=[connection.DEFAULT_SERVER]):
     for server in servers_list:
         try:
             self._sm_client = SystemManager(server)
             return
         except:
             pass
     if self._sm_client is None:
         raise NoServerAvailable('Cannot create system_manager connections on server list %s' % str(servers_list))
def setup_module():
    global pool, cf, scf, counter_cf, super_counter_cf, sysman
    credentials = {'username': '******', 'password': '******'}
    pool = ConnectionPool(keyspace='PycassaTestKeyspace', credentials=credentials)
    cf = ColumnFamily(pool, 'Standard1')
    scf = ColumnFamily(pool, 'Super1')
    sysman = SystemManager()
    counter_cf = ColumnFamily(pool, 'Counter1')
    super_counter_cf = ColumnFamily(pool, 'SuperCounter1')
    def _cassandra_ensure_keyspace(self, server_list, keyspace_name,
                                   cf_info_list):
        # Retry till cassandra is up
        server_idx = 0
        num_dbnodes = len(self._server_list)
        connected = False
        while not connected:
            try:
                cass_server = self._server_list[server_idx]
                sys_mgr = SystemManager(cass_server)
                connected = True
            except Exception as e:
                # TODO do only for
                # thrift.transport.TTransport.TTransportException
                server_idx = (server_idx + 1) % num_dbnodes
                time.sleep(3)

        if self._reset_config:
            try:
                sys_mgr.drop_keyspace(keyspace_name)
            except pycassa.cassandra.ttypes.InvalidRequestException as e:
                # TODO verify only EEXISTS
                self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)

        try:
            sys_mgr.create_keyspace(keyspace_name, SIMPLE_STRATEGY,
                                    {'replication_factor': str(num_dbnodes)})
        except pycassa.cassandra.ttypes.InvalidRequestException as e:
            # TODO verify only EEXISTS
            self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)

        for cf_info in cf_info_list:
            try:
                (cf_name, comparator_type) = cf_info
                if comparator_type:
                    sys_mgr.create_column_family(
                        keyspace_name,
                        cf_name,
                        comparator_type=comparator_type)
                else:
                    sys_mgr.create_column_family(keyspace_name, cf_name)
            except pycassa.cassandra.ttypes.InvalidRequestException as e:
                # TODO verify only EEXISTS
                self._logger("Warning! " + str(e), level=SandeshLevel.SYS_WARN)
 def ensure_cassandra_cf(self):
     s = SystemManager()
     if self.keyspace not in s.list_keyspaces():
         s.create_keyspace(self.keyspace, SIMPLE_STRATEGY,
                           {'replication_factor': '1'})
     if self.cassandra_columns_family not in s.get_keyspace_column_families(
             self.keyspace):
         s.create_column_family(self.keyspace,
                                self.cassandra_columns_family)
     self.columnfamily = ColumnFamily(self.cassandra_session,
                                      self.cassandra_columns_family)
Exemple #41
0
def detect_nodes(seeds, keyspace):
    from feedly import settings
    if not settings.FEEDLY_DISCOVER_CASSANDRA_NODES:
        logger.warning('cassandra nodes discovery is off')
        return seeds
    nodes = frozenset(seeds)
    logging.info('retrieve nodes from seeds %r' % seeds)
    for seed in seeds:
        try:
            sys_manager = SystemManager(seed)
        except TTransportException:
            logging.warning('%s is not a seed or is not reachable' % seed)
            continue
        ring_description = sys_manager.describe_ring(keyspace)
        for ring_range in ring_description:
            endpoint_details = ring_range.endpoint_details[0]
            hostname = endpoint_details.host
            port = getattr(endpoint_details, 'port', 9160)
            nodes = nodes.union({'%s:%s' % (hostname, port), }, nodes)
        break
    return nodes
Exemple #42
0
 def create_cfs(self):
     """
     Creates the Cassandra Column Families (if not exist)
     """
     sys_mgr = None
     pool = None
     try:
         sys_mgr = SystemManager()
         pool = ConnectionPool(settings.KEYSPACE,
                               server_list=settings.CASSANDRA_HOSTS)
         for cf_name in [
                 CF_LOGS, CF_LOGS_BY_APP, CF_LOGS_BY_HOST,
                 CF_LOGS_BY_SEVERITY
         ]:
             try:
                 cf = ColumnFamily(pool, cf_name)
             except:
                 logger.info("create_cfs(): Creating column family %s",
                             cf_name)
                 sys_mgr.create_column_family(
                     settings.KEYSPACE,
                     cf_name,
                     comparator_type=TimeUUIDType())
                 cf = ColumnFamily(pool, cf_name)
                 cf.get_count(str(uuid.uuid4()))
     finally:
         if pool:
             pool.dispose()
         if sys_mgr:
             sys_mgr.close()
Exemple #43
0
def detect_nodes(seeds, keyspace):
    from feedly import settings
    if not settings.FEEDLY_DISCOVER_CASSANDRA_NODES:
        logger.warning('cassandra nodes discovery is off')
        return seeds
    nodes = frozenset(seeds)
    logging.info('retrieve nodes from seeds %r' % seeds)
    for seed in seeds:
        try:
            sys_manager = SystemManager(seed)
        except TTransportException:
            logging.warning('%s is not a seed or is not reachable' % seed)
            continue
        ring_description = sys_manager.describe_ring(keyspace)
        for ring_range in ring_description:
            endpoint_details = ring_range.endpoint_details[0]
            hostname = endpoint_details.host
            port = getattr(endpoint_details, 'port', 9160)
            nodes = nodes.union({
                '%s:%s' % (hostname, port),
            }, nodes)
        break
    return nodes
Exemple #44
0
def cassandra_reset():
    from feedly import settings
    hostname = settings.FEEDLY_CASSANDRA_HOSTS[0]
    keyspace = 'test_feedly'

    sys = SystemManager(hostname)

    # sys.drop_keyspace(keyspace)

    if keyspace not in sys.list_keyspaces():
        sys.create_keyspace(keyspace, SIMPLE_STRATEGY,
                            {'replication_factor': '1'})

        sys.create_column_family(keyspace,
                                 'activity',
                                 comparator_type=UTF8_TYPE)

        sys.create_column_family(keyspace,
                                 'timeline',
                                 comparator_type=IntegerType(reversed=True))
def createTSColumnFamily(servers, keyspace, tableName, credentials=None):
  """Create a tsXX Column Family using one of the servers in the ``servers``
  list and the ``keysapce`` and ``tableName``.
  """
  for server in servers:
    try:
      manager = SystemManager(server, credentials=credentials)
      createUTF8ColumnFamily(manager, keyspace, tableName, ts_table=True)
      return None
    except (Exception) as e:
      # TODO: log when we know how to log
      lastError = e

  raise RuntimeError("Failed to create CF %s.%s using the server list %s, "\
    "last error was %s" % (keyspace, tableName, servers, str(lastError)))
Exemple #46
0
 def create_keyspace(self):
     """
     Creates the Cassandra Keyspace (if not exist)
     """
     sys_mgr = None
     try:
         sys_mgr = SystemManager()
         try:
             sys_mgr.describe_ring(settings.KEYSPACE)
         except:
             logger.info("create_keyspace(): Creating keyspace %s",
                         settings.KEYSPACE)
             sys_mgr.create_keyspace(settings.KEYSPACE, SIMPLE_STRATEGY,
                                     {'replication_factor': '1'})
     finally:
         if sys_mgr:
             sys_mgr.close()
 def _cassandra_system_manager(self):
     # Retry till cassandra is up
     server_idx = 0
     connected = False
     while not connected:
         try:
             cass_server = self._server_list[server_idx]
             sys_mgr = SystemManager(cass_server, credentials=self._credential)
             connected = True
         except Exception:
             # TODO do only for
             # thrift.transport.TTransport.TTransportException
             server_idx = (server_idx + 1) % self._num_dbnodes
             time.sleep(3)
     return sys_mgr
Exemple #48
0
 def __init__(self, engine, **kw):
     super(CassandraStore, self).__init__(engine, **kw)
     spliturl = urlparse.urlsplit(engine)
     _, keyspace, column_family = spliturl[2].split('/')
     try:
         self._pool = pycassa.connect(keyspace, [spliturl[1]])
         self._store = pycassa.ColumnFamily(self._pool, column_family)
     except pycassa.InvalidRequestException:
         from pycassa.system_manager import SystemManager
         system_manager = SystemManager(spliturl[1])
         system_manager.create_keyspace(keyspace, kw.get('replication', 1))
         system_manager.create_column_family(keyspace, column_family)
         self._pool = pycassa.connect(keyspace, [spliturl[1]])
         self._store = pycassa.ColumnFamily(self._pool, column_family)
    def _create_column_family(self,
                              family,
                              bytes_columns=[],
                              key_validation_class=TIME_UUID_TYPE):
        '''
        Creates a column family of the name 'family' and sets any of
        the names in the bytes_column list to have the BYTES_TYPE.

        key_validation_class defaults to TIME_UUID_TYPE and could also
        be ASCII_TYPE for md5 hash keys, like we use for 'inbound'
        '''
        sm = SystemManager(random.choice(self.server_list))
        # sys.create_column_family(self.namespace, family, super=False)
        sm.create_column_family(self.namespace,
                                family,
                                super=False,
                                key_validation_class=key_validation_class,
                                default_validation_class=TIME_UUID_TYPE,
                                column_name_class=ASCII_TYPE)
        for column in bytes_columns:
            sm.alter_column(self.namespace, family, column, BYTES_TYPE)
        sm.close()
def _init_db(server):

    dbsys = SystemManager(server)
    dbsys.create_keyspace(KEY_SPACE, SIMPLE_STRATEGY,
                          {"replication_factor": "1"})

    column_family_type = types.CompositeType(
        types.UTF8Type(),
        types.UTF8Type(),
    )

    dbsys.create_column_family(
        KEY_SPACE,
        COLUMN_FAMILY,
        comparator_type=column_family_type,
        key_validation_class=types.UTF8Type(),
        default_validation_class=types.UTF8Type(),
    )
 def __init__(self, engine, **kw):
     super(CassandraStore, self).__init__(engine, **kw)
     spliturl = urlsplit(engine)
     _, keyspace, column_family = spliturl.path.split('/')
     try:
         self._store = pycassa.ColumnFamily(
             pycassa.ConnectionPool(keyspace, [spliturl.hostname]),
             column_family,
         )
     except pycassa.InvalidRequestException:
         from pycassa.system_manager import SystemManager  # @UnresolvedImport @IgnorePep8
         system_manager = SystemManager(spliturl[1])
         system_manager.create_keyspace(
             keyspace,
             pycassa.system_manager.SIMPLE_STRATEGY,
             dict(replication_factor=native(kw.get('replication', 1))),
         )
         system_manager.create_column_family(keyspace, column_family)
         self._store = pycassa.ColumnFamily(
             pycassa.ConnectionPool(keyspace, [spliturl.netloc]),
             column_family,
         )
def _clean_db(server):

    dbsys = SystemManager(server)
    dbsys.drop_keyspace(KEY_SPACE)
    dbsys.close()
Exemple #53
0
def _sys_mgr():
    '''
    Return a pycassa system manager connection object
    '''
    return SystemManager('{0}:{1}'.format(host, thrift_port))
    def db_export(self):
        db_contents = {'cassandra': {},
                       'zookeeper': {}}

        cassandra_contents = db_contents['cassandra']
        creds = None
        if self._api_args.cassandra_user and self._api_args.cassandra_password:
            creds = {'username': self._api_args.cassandra_user,
                     'password': self._api_args.cassandra_password}
        socket_factory = default_socket_factory
        if ('cassandra_use_ssl' in self._api_args and
                self._api_args.cassandra_use_ssl):
            socket_factory = self._make_ssl_socket_factory(
                self._api_args.cassandra_ca_certs, validate=False)
        sys_mgr = SystemManager(
            self._api_args.cassandra_server_list[0],
            credentials=creds,
            socket_factory=socket_factory)
        existing_keyspaces = sys_mgr.list_keyspaces()
        for ks_name in set(KEYSPACES) - set(self._args.omit_keyspaces or []):
            if self._api_args.cluster_id:
                full_ks_name = '%s_%s' %(self._api_args.cluster_id, ks_name)
            else:
                full_ks_name = ks_name
            if full_ks_name not in existing_keyspaces:
                continue
            cassandra_contents[ks_name] = {}

            pool = ConnectionPool(
                full_ks_name, self._api_args.cassandra_server_list,
                pool_timeout=120, max_retries=-1, timeout=5,
                socket_factory=socket_factory, credentials=creds)
            for cf_name in sys_mgr.get_keyspace_column_families(full_ks_name):
                cassandra_contents[ks_name][cf_name] = {}
                cf = ColumnFamily(pool, cf_name,
                                  buffer_size=self._args.buffer_size)
                for r,c in cf.get_range(column_count=10000000, include_timestamp=True):
                    cassandra_contents[ks_name][cf_name][r] = c
        logger.info("Cassandra DB dumped")

        def get_nodes(path):
            if path[:-1].rpartition('/')[-1] in self._zk_ignore_list:
                return []

            try:
                if not zk.get_children(path):
                    return [(path, zk.get(path))]
            except kazoo.exceptions.NoNodeError:
                return []

            nodes = []
            for child in zk.get_children(path):
                nodes.extend(get_nodes('%s%s/' %(path, child)))

            return nodes

        zk = kazoo.client.KazooClient(self._api_args.zk_server_ip)
        zk.start()
        nodes = get_nodes(self._api_args.cluster_id+'/')
        zk.stop()
        db_contents['zookeeper'] = json.dumps(nodes)
        logger.info("Zookeeper DB dumped")

        f = open(self._args.export_to, 'w')
        try:
            f.write(json.dumps(db_contents))
        finally:
            f.close()
        logger.info("DB dump wrote to file %s" % self._args.export_to)
def setup_module():
    global sys
    sys = SystemManager()
Exemple #56
0
    def create_cfs(self):
        """
        Creates the Cassandra Column Families (if not exist)
        """
        sys_mgr = None
        pool = None
        try:
            sys_mgr = SystemManager()
            pool = ConnectionPool(settings.KEYSPACE,
                                  server_list=settings.CASSANDRA_HOSTS)

            try:
                cf = ColumnFamily(pool, CF_LOGS)
            except:
                logger.info("create_cfs(): Creating column family %s", CF_LOGS)
                #========================================
                # Column key -> CompositeType
                #========================================
                # 1. UUID + Timestamp
                # 2. Host / Origin
                # 3. Application
                # 4. Severiry
                comparator = CompositeType(TimeUUIDType(), UTF8Type(),
                                           UTF8Type(), UTF8Type())
                sys_mgr.create_column_family(settings.KEYSPACE,
                                             CF_LOGS,
                                             comparator_type=comparator)
                cf = ColumnFamily(pool, CF_LOGS)
                # cf.get_count(str(uuid.uuid4()))

            try:
                cf = ColumnFamily(pool, CF_METADATA)
            except:
                logger.info("create_cfs(): Creating column family %s",
                            CF_METADATA)
                sys_mgr.create_column_family(settings.KEYSPACE,
                                             CF_METADATA,
                                             comparator_type=UTF8Type())
                cf = ColumnFamily(pool, CF_METADATA)
                cf.get_count(str(uuid.uuid4()))

            try:
                cf = ColumnFamily(pool, CF_TIMESTAMP_BITMAP)
            except:
                logger.info("create_cfs(): Creating column family %s",
                            CF_TIMESTAMP_BITMAP)
                sys_mgr.create_column_family(settings.KEYSPACE,
                                             CF_TIMESTAMP_BITMAP,
                                             comparator_type=IntegerType())
                cf = ColumnFamily(pool, CF_TIMESTAMP_BITMAP)

            try:
                cf = ColumnFamily(pool, CF_MULTI_MESSAGELOGS)
            except:
                logger.info("create_cfs(): Creating column family %s",
                            CF_MULTI_MESSAGELOGS)
                sys_mgr.create_column_family(settings.KEYSPACE,
                                             CF_MULTI_MESSAGELOGS,
                                             comparator_type=UTF8Type())
                cf = ColumnFamily(pool, CF_MULTI_MESSAGELOGS)

                sys_mgr.create_index(settings.KEYSPACE,
                                     CF_MULTI_MESSAGELOGS,
                                     'meta:host',
                                     UTF8_TYPE,
                                     index_name='multimsg_host_index')
                sys_mgr.create_index(settings.KEYSPACE,
                                     CF_MULTI_MESSAGELOGS,
                                     'meta:application',
                                     UTF8_TYPE,
                                     index_name='multimsg_application_index')
                sys_mgr.create_index(settings.KEYSPACE,
                                     CF_MULTI_MESSAGELOGS,
                                     'meta:status',
                                     UTF8_TYPE,
                                     index_name='multimsg_finish_status_index')

        finally:
            if pool:
                pool.dispose()
            if sys_mgr:
                sys_mgr.close()
Exemple #57
0
class CCli(Cmd, object):
    prompt = '> '
    continuation_prompt = '. '
    
    timing = True
    colors = True
    debug = True

    # O'Key, guys, you created good library with a lot of parameters, but,
    # I think, you should test it before release, shouldn't?
    case_insensitive = False

    max_data_size = 35
    max_rows = 50
    
    def __init__(self, *args, **kwargs):
        super(CCli, self).__init__(*args, **kwargs)

        self.sm = None
        self.pool = None

        self.server = None
        self.keyspace = None

        self.settable['max_data_size'] = 'Maximum value symbols [0 = no truncating]'
        self.settable['max_rows'] = 'Maximum rows to receive by one get'

    def func_named(self, arg):
        return super(CCli, self).func_named(arg[0])

    @parse(Optional(server, default='locahost:9160'))
    def do_connect(self, server):
        try:
            self.sm = SystemManager(server)
        except TTransportException:
            return self.perror('Can not connect to %s' % server)
        
        self.server = server
        self.prompt = '%s> ' % server

        self.keyspace = None
        self.pool = None

        print 'Successfully connected to %s' % server

    @check_connection
    @parse(keyspace)
    def do_use(self, keyspace):
        if keyspace not in self.sm.list_keyspaces():
            return self.perror('Unknown keyspace %s' % keyspace)

        self.prompt = '%s/%s> ' % (self.server, keyspace)
        self.keyspace = keyspace
        self.pool = ConnectionPool(keyspace, server_list=[self.server])
        
        print 'Using %s as default keyspace' % keyspace

    def complete_use(self, text, line, begidx, endidx):
        if not self.sm:
            return []
        
        return [x for x in self.sm.list_keyspaces() if x.startswith(text)]

    @check_connection
    @parse(Keyword('keyspaces') | Keyword('columnfamilies'))
    def do_list(self, space):
        return getattr(self, 'list_%s' % space)()

    def list_keyspaces(self):
        pt = PrettyTable()
        pt.field_names = ['Keyspaces']
        pt.align = 'l'

        for ks in self.sm.list_keyspaces():
            pt.add_row([ks])

        print pt.get_string(sortby="Keyspaces")

    @check_keyspace
    def list_columnfamilies(self):
        pt = PrettyTable()
        pt.field_names = ['ColumnFamilies']
        pt.align = 'l'

        for cf in self.sm.get_keyspace_column_families(self.keyspace).keys():
            pt.add_row([cf])

        print pt.get_string(sortby="ColumnFamilies")

    def complete_list(self, text, line, begidx, endidx):
        return [x for x in ['keyspaces', 'columnfamilies'] if x.startswith(text)]

    @check_connection
    @parse(Keyword('keyspace') + Optional(keyspace, default=None) | Keyword('columnfamily') + columnfamily)
    def do_describe(self, space, name):
        return getattr(self, 'describe_%s' % space)(name)

    def describe_keyspace(self, keyspace):
        keyspace = keyspace or self.keyspace
        if not keyspace:
            return self.perror('Please, select working keyspace or define it as command parameter')

        try:
            options = self.sm.get_keyspace_properties(keyspace)
        except NotFoundException:
            return self.perror('Unknown keyspace %s' % keyspace)

        pt = PrettyTable()
        pt.field_names = ['Keyspace', keyspace]
        pt.align["Keyspace"] = "l"
        pt.align[keyspace] = 'r'

        pt.add_row(['replication_strategy', options['replication_strategy']])
        for k, v in options['strategy_options'].items():
            pt.add_row([k, "\033[34m%s\033[0m" % v])

        print pt.get_string(sortby='Keyspace')

    @check_keyspace
    def describe_columnfamily(self, columnfamily):
        try:
            options = self.sm.get_keyspace_column_families(self.keyspace, use_dict_for_col_metadata=True)[columnfamily]
        except KeyError:
            return self.perror('Unknown columnfamily %s' % columnfamily)

        pt = PrettyTable()
        pt.field_names = ['ColumnFamily', columnfamily]
        pt.align["ColumnFamily"] = "l"
        pt.align[columnfamily] = 'r'

        for k, v in options.__dict__.items():
            if k == 'column_metadata' and len(v):
                continue
            pt.add_row([k, v])

        print pt.get_string(sortby='ColumnFamily')

        if len(options.column_metadata):
            pt = PrettyTable()
            pt.field_names = ['Column \ Options'] + options.column_metadata.values()[0].__dict__.keys()

            for k, v in options.column_metadata.items():
                pt.add_row([k] + v.__dict__.values())

            print pt.get_string(sortby='Column \ Options')

    def complete_describe(self, text, line, begidx, endidx):
        return [x for x in ['keyspace', 'columnfamily'] if x.startswith(text)]

    def completenames(self, text, *ignored):
        names = super(CCli, self).completenames(text, *ignored)
        if self.keyspace:
            names.extend(cf for cf in
                self.sm.get_keyspace_column_families(self.keyspace).keys()
                if cf.startswith(text))

        return names

    def default(self, line):
        # Ugly, ugly things...
        line = (' '.join(line.parsed)).strip()

        if not self.server and not self.keyspace:
            return super(CCli, self).default(line)
        return self.simple_select(line)

    @parse(columnfamily + Optional('[' + Combine(Optional(key, default='') +
        Optional(':' + Optional(key, default='') + Optional(':' +
        Optional(count, default='')))) + ']')
    )
    def simple_select(self, columnfamily, *args):
        slice = ['', '', self.max_rows]
        key = None
    
        if args and args[1]:
            if ':' not in args[1]:
                key = args[1]
            for i, part in enumerate(args[1].split(':', 2)):
                slice[i] = part

        try:
            cf = ColumnFamily(self.pool, columnfamily)
        except NotFoundException:
            return super(CCli, self).default(' '.join([columnfamily] + list(args)))

        if key:
            pt = PrettyTable()
            pt.field_names = ['Key', key]
            pt.align["Key"] = "l"
            pt.align[key] = 'r'

            for k, v in cf.get(key).items():
                pt.add_row([k, (v[:self.max_data_size - 3] + '...' if self.max_data_size and len(v) > self.max_data_size else v)])

            print pt.get_string(sortby='Key')
            return

        data = dict(cf.get_range(start=slice[0], finish=slice[1], row_count=int(slice[2])))

        columns = []
        for key, row in data.items():
            columns.extend(row.keys())
        columns = list(set(columns))
        columns.sort()

        pt = PrettyTable()
        pt.field_names = ['Key / Column'] + columns
        pt.align["Key / Column"] = "l"
        for column in columns:
            pt.align[column] = "r"

        for key, row in data.items():
            prow = [key]
            for column in columns:
                value = row.get(column, '---')
                if len(value) > self.max_data_size:
                    value = value[:self.max_data_size - 3] + '...'
                    
                prow.append(value)
            pt.add_row(prow)

        print pt.get_string(sortby='Key / Column')