def __init__(self, name, fields):
"""
init a data space
:param name:
:param fields: [b'A', b'B']
"""
self.db_loc = './db'
self.name=name
opts = rocksdb.Options(create_if_missing=True)
# self.columns = {}
if os.path.exists(os.path.join(self.db_loc, name)):
column_families = {}
if fields is None or len(fields)==0:
fields=get_columns(self.db_loc, name)
for fld in fields:
column_families[fld] = rocksdb.ColumnFamilyOptions()
self.db = rocksdb.DB(os.path.join(self.db_loc, name), opts, column_families=column_families)
# self.columns = column_families
else:
self.db = rocksdb.DB(os.path.join(self.db_loc, name), opts)
for fld in fields:
# fld format: b'A'
# self.columns[fld]=(self.db.create_column_family(fld, rocksdb.ColumnFamilyOptions()))
self.db.create_column_family(fld, rocksdb.ColumnFamilyOptions())
def setUp(self):
TestHelper.setUp(self)
opts = rocksdb.Options(create_if_missing=True)
self.db = rocksdb.DB(
os.path.join(self.db_loc, 'test'),
opts,
)
self.cf_a = self.db.create_column_family(b'A',
rocksdb.ColumnFamilyOptions())
self.cf_b = self.db.create_column_family(b'B',
rocksdb.ColumnFamilyOptions())
def _fresh_cf(self, cf_name: bytes) -> 'rocksdb.ColumnFamilyHandle':
"""Ensure we have a working and fresh column family"""
import rocksdb
log_cf = self.log.new(cf=cf_name.decode('ascii'))
_cf = self._db.get_column_family(cf_name)
# XXX: dropping column because initialization currently expects a fresh index
if _cf is not None:
old_id = _cf.id
log_cf.debug('drop existing column family')
self._db.drop_column_family(_cf)
else:
old_id = None
log_cf.debug('no need to drop column family')
del _cf
log_cf.debug('create fresh column family')
_cf = self._db.create_column_family(cf_name,
rocksdb.ColumnFamilyOptions())
new_id = _cf.id
assert _cf is not None
assert _cf.is_valid
assert new_id != old_id
log_cf.debug('got column family',
is_valid=_cf.is_valid,
id=_cf.id,
old_id=old_id)
return _cf
def _get_or_create_column_family(self,
cf_name: bytes) -> Any: # TODO: typing
import rocksdb
cf = self._db.get_column_family(cf_name)
if cf is None:
cf = self._db.create_column_family(cf_name,
rocksdb.ColumnFamilyOptions())
return cf
def column_family_options(comparator: Optional[Comparator] = None, prefix_extractor: Optional[SliceTransform] = None, merge_operator: Optional[Union[MergeOperator, AssociativeMergeOperator]] = None) -> rocksdb.ColumnFamilyOptions: opts = rocksdb.ColumnFamilyOptions() if comparator is not None: opts.comparator = comparator if prefix_extractor is not None: opts.prefix_extractor = prefix_extractor if merge_operator is not None: opts.merge_operator = merge_operator return opts
def __init__(self,
path: str = './',
with_index: bool = True,
cache_capacity: Optional[int] = None,
use_memory_indexes: bool = False):
import rocksdb
self.log = logger.new()
self._path = path
self._use_memory_indexes = use_memory_indexes
tx_dir = os.path.join(path, _DB_NAME)
lru_cache = cache_capacity and rocksdb.LRUCache(cache_capacity)
table_factory = rocksdb.BlockBasedTableFactory(block_cache=lru_cache)
options = rocksdb.Options(
table_factory=table_factory,
write_buffer_size=83886080, # 80MB (default is 4MB)
compression=rocksdb.CompressionType.no_compression,
allow_mmap_writes=True, # default is False
allow_mmap_reads=True, # default is already True
)
cf_names: List[bytes]
try:
# get the list of existing column families
cf_names = rocksdb.list_column_families(tx_dir, options)
except rocksdb.errors.RocksIOError:
# this means the db doesn't exist, a repair will create one
rocksdb.repair_db(tx_dir, options)
cf_names = []
# we need to open all column families
column_families = {
cf: rocksdb.ColumnFamilyOptions()
for cf in cf_names
}
# finally, open the database
self._db = rocksdb.DB(tx_dir, options, column_families=column_families)
self.log.debug('open db',
cf_list=[
cf.name.decode('ascii')
for cf in self._db.column_families
])
self._cf_tx = self._get_or_create_column_family(_CF_NAME_TX)
self._cf_meta = self._get_or_create_column_family(_CF_NAME_META)
self._cf_attr = self._get_or_create_column_family(_CF_NAME_ATTR)
super().__init__(with_index=with_index)
import rocksdb
options = rocksdb.Options()
options.create_if_missing = True
db = rocksdb.DB("rocksdb_column_families_example.db", options)
cf = db.create_column_family(b"new_cf", rocksdb.ColumnFamilyOptions())
del db
column_families = {}
# kDefaultColumnFamilyName == 'default'
column_families[b"default"] = rocksdb.ColumnFamilyOptions()
column_families[b"new_cf"] = rocksdb.ColumnFamilyOptions()
db = rocksdb.DB("rocksdb_column_families_example.db",
options,
column_families=column_families)
# e.g) db.column_families
# [<ColumnFamilyHandle name: b'default', id: 0, state: valid>, <ColumnFamilyHandle name: b'cf', id: 1, state: valid>]
default_column_family = db.get_column_family(b'default')
new_cf_column_family = db.get_column_family(b'new_cf')
db.put((new_cf_column_family, b'key'), b'value')
print(db.get((new_cf_column_family, b'key'))) # b'value'
batch = rocksdb.WriteBatch()
def add_col(self, fld):
self.db.create_column_family(fld, rocksdb.ColumnFamilyOptions())