def _vid_for_value(self, column, key):
if column.is_trie:
if column.rtrie_indicator == mdb.MDB_UINT_16:
key = rtrie.vid_for_value(self.vid16_nodes, self.vid16_kids, key)
else:
key = rtrie.vid_for_value(self.vid_nodes, self.vid_kids, key)
elif column.is_lz4:
key = clz4.compress(key)
return key
def _vid_for_value(self, column, key):
if column.is_trie:
if column.rtrie_indicator == mdb.MDB_UINT_16:
key = rtrie.vid_for_value(self.vid16_nodes, self.vid16_kids, key)
else:
key = rtrie.vid_for_value(self.vid_nodes, self.vid_kids, key)
elif column.is_lz4:
key = clz4.compress(key)
return key
def test_marble_insert(self):
# test general infomation
self.assertEqual(self.n_inserted, len(_ALBUMS))
self.assertEqual(_NPARTITIONS, len(self.files))
part_id = {}
# test that each sub db is fine
for date, file in self.files.iteritems():
env, txn, dbs, meta = self.marble._open(file)
# check meta db
self.assertTrue(meta.contains(txn, "_vid_nodes"))
self.assertTrue(meta.contains(txn, "_vid_kids"))
self.assertTrue(meta.contains(txn, "_vid16_nodes"))
self.assertTrue(meta.contains(txn, "_vid16_kids"))
self.assertEqual(meta.get(txn, "name"), ujson.dumps("Collections"))
self.assertEqual(meta.get(txn, "partition"), ujson.dumps("date"))
self.assertEqual(meta.get(txn, "fields"), ujson.dumps(_FIELDS))
self.assertEqual(meta.get(txn, "_pdata"), ujson.dumps(date))
vid_nodes, _ = meta.get_raw(txn, '_vid_nodes')
vid_kids, _ = meta.get_raw(txn, '_vid_kids')
vid16_nodes, _ = meta.get_raw(txn, '_vid16_nodes', (None, 0))
vid16_kids, _ = meta.get_raw(txn, '_vid16_kids', (None, 0))
# check subdb, subinddb
part_id[date] = 1
for name, (db, ind_db, _, column, _) in dbs.iteritems():
if name == "_count":
continue
bitmaps = {}
part_id[date] = 1
for album in self.albums:
if date == album[_PARTITIONS]: # match the partition
value = album[name]
i = part_id[album[_PARTITIONS]]
part_id[album[_PARTITIONS]] += 1
if column.is_trie:
if column.rtrie_indicator == mdb.MDB_UINT_16:
val = rtrie.vid_for_value(
vid16_nodes, vid16_kids, value)
else:
val = rtrie.vid_for_value(
vid_nodes, vid_kids, value)
elif column.is_lz4:
val = clz4.compress(value)
else:
val = value
# self.assertEqual(db.get(txn, i), val)
if ind_db is not None:
# row_id should be in bitmap too
if val in bitmaps:
bitmap = bitmaps[val]
else:
bitmap = BitSet()
bitmap.loads(ind_db.get(txn, val))
bitmaps[val] = bitmap
self.assertTrue(i in bitmap)
txn.commit()
env.close()
def test_marble_insert(self):
# test general infomation
self.assertEqual(self.n_inserted, len(_ALBUMS))
self.assertEqual(_NPARTITIONS, len(self.files))
part_id = {}
# test that each sub db is fine
for date, file in self.files.iteritems():
env, txn, dbs, meta = self.marble._open(file)
# check meta db
self.assertTrue(meta.contains(txn, "_vid_nodes"))
self.assertTrue(meta.contains(txn, "_vid_kids"))
self.assertTrue(meta.contains(txn, "_vid16_nodes"))
self.assertTrue(meta.contains(txn, "_vid16_kids"))
self.assertEqual(meta.get(txn, "name"), ujson.dumps("Collections"))
self.assertEqual(meta.get(txn, "partition"), ujson.dumps("date"))
self.assertEqual(meta.get(txn, "fields"), ujson.dumps(_FIELDS))
self.assertEqual(meta.get(txn, "_pdata"), ujson.dumps(date))
vid_nodes, _ = meta.get_raw(txn, '_vid_nodes')
vid_kids, _ = meta.get_raw(txn, '_vid_kids')
vid16_nodes, _ = meta.get_raw(txn, '_vid16_nodes', (None, 0))
vid16_kids, _ = meta.get_raw(txn, '_vid16_kids', (None, 0))
# check subdb, subinddb
part_id[date] = 1
for name, (db, ind_db, _, column, _) in dbs.iteritems():
if name == "_count":
continue
bitmaps = {}
part_id[date] = 1
for album in self.albums:
if date == album[_PARTITIONS]: # match the partition
value = album[name]
i = part_id[album[_PARTITIONS]]
part_id[album[_PARTITIONS]] += 1
if column.is_trie:
if column.rtrie_indicator == mdb.MDB_UINT_16:
val = rtrie.vid_for_value(vid16_nodes, vid16_kids, value)
else:
val = rtrie.vid_for_value(vid_nodes, vid_kids, value)
elif column.is_lz4:
val = clz4.compress(value)
else:
val = value
# self.assertEqual(db.get(txn, i), val)
if ind_db is not None:
# row_id should be in bitmap too
if val in bitmaps:
bitmap = bitmaps[val]
else:
bitmap = BitSet()
bitmap.loads(ind_db.get(txn, val))
bitmaps[val] = bitmap
self.assertTrue(i in bitmap)
txn.commit()
env.close()
def _convert_lz4(val, vid_trie=None, vid_trie16=None):
try:
val = str(val)
except:
val = val.encode('utf-8')
return clz4.compress(val)
def _convert_lz4(val, vid_trie=None, vid_trie16=None):
try:
val = str(val)
except:
val = val.encode('utf-8')
return clz4.compress(val)
