Esempio n. 1
0
 def _vid_for_value(self, column, key):
     if column.is_trie:
         if column.rtrie_indicator == mdb.MDB_UINT_16:
             key = rtrie.vid_for_value(self.vid16_nodes, self.vid16_kids, key)
         else:
             key = rtrie.vid_for_value(self.vid_nodes, self.vid_kids, key)
     elif column.is_lz4:
         key = clz4.compress(key)
     return key
Esempio n. 2
0
 def _vid_for_value(self, column, key):
     if column.is_trie:
         if column.rtrie_indicator == mdb.MDB_UINT_16:
             key = rtrie.vid_for_value(self.vid16_nodes, self.vid16_kids, key)
         else:
             key = rtrie.vid_for_value(self.vid_nodes, self.vid_kids, key)
     elif column.is_lz4:
         key = clz4.compress(key)
     return key
Esempio n. 3
0
 def test_marble_insert(self):
     #  test general infomation
     self.assertEqual(self.n_inserted, len(_ALBUMS))
     self.assertEqual(_NPARTITIONS, len(self.files))
     part_id = {}
     #  test that each sub db is fine
     for date, file in self.files.iteritems():
         env, txn, dbs, meta = self.marble._open(file)
         #  check meta db
         self.assertTrue(meta.contains(txn, "_vid_nodes"))
         self.assertTrue(meta.contains(txn, "_vid_kids"))
         self.assertTrue(meta.contains(txn, "_vid16_nodes"))
         self.assertTrue(meta.contains(txn, "_vid16_kids"))
         self.assertEqual(meta.get(txn, "name"), ujson.dumps("Collections"))
         self.assertEqual(meta.get(txn, "partition"), ujson.dumps("date"))
         self.assertEqual(meta.get(txn, "fields"), ujson.dumps(_FIELDS))
         self.assertEqual(meta.get(txn, "_pdata"), ujson.dumps(date))
         vid_nodes, _ = meta.get_raw(txn, '_vid_nodes')
         vid_kids, _ = meta.get_raw(txn, '_vid_kids')
         vid16_nodes, _ = meta.get_raw(txn, '_vid16_nodes', (None, 0))
         vid16_kids, _ = meta.get_raw(txn, '_vid16_kids', (None, 0))
         #  check subdb, subinddb
         part_id[date] = 1
         for name, (db, ind_db, _, column, _) in dbs.iteritems():
             if name == "_count":
                 continue
             bitmaps = {}
             part_id[date] = 1
             for album in self.albums:
                 if date == album[_PARTITIONS]:  # match the partition
                     value = album[name]
                     i = part_id[album[_PARTITIONS]]
                     part_id[album[_PARTITIONS]] += 1
                     if column.is_trie:
                         if column.rtrie_indicator == mdb.MDB_UINT_16:
                             val = rtrie.vid_for_value(
                                 vid16_nodes, vid16_kids, value)
                         else:
                             val = rtrie.vid_for_value(
                                 vid_nodes, vid_kids, value)
                     elif column.is_lz4:
                         val = clz4.compress(value)
                     else:
                         val = value
                     # self.assertEqual(db.get(txn, i), val)
                     if ind_db is not None:
                         #  row_id should be in bitmap too
                         if val in bitmaps:
                             bitmap = bitmaps[val]
                         else:
                             bitmap = BitSet()
                             bitmap.loads(ind_db.get(txn, val))
                             bitmaps[val] = bitmap
                         self.assertTrue(i in bitmap)
         txn.commit()
         env.close()
Esempio n. 4
0
 def test_marble_insert(self):
     #  test general infomation
     self.assertEqual(self.n_inserted, len(_ALBUMS))
     self.assertEqual(_NPARTITIONS, len(self.files))
     part_id = {}
     #  test that each sub db is fine
     for date, file in self.files.iteritems():
         env, txn, dbs, meta = self.marble._open(file)
         #  check meta db
         self.assertTrue(meta.contains(txn, "_vid_nodes"))
         self.assertTrue(meta.contains(txn, "_vid_kids"))
         self.assertTrue(meta.contains(txn, "_vid16_nodes"))
         self.assertTrue(meta.contains(txn, "_vid16_kids"))
         self.assertEqual(meta.get(txn, "name"), ujson.dumps("Collections"))
         self.assertEqual(meta.get(txn, "partition"), ujson.dumps("date"))
         self.assertEqual(meta.get(txn, "fields"), ujson.dumps(_FIELDS))
         self.assertEqual(meta.get(txn, "_pdata"), ujson.dumps(date))
         vid_nodes, _ = meta.get_raw(txn, '_vid_nodes')
         vid_kids, _ = meta.get_raw(txn, '_vid_kids')
         vid16_nodes, _ = meta.get_raw(txn, '_vid16_nodes', (None, 0))
         vid16_kids, _ = meta.get_raw(txn, '_vid16_kids', (None, 0))
         #  check subdb, subinddb
         part_id[date] = 1
         for name, (db, ind_db, _, column, _) in dbs.iteritems():
             if name == "_count":
                 continue
             bitmaps = {}
             part_id[date] = 1
             for album in self.albums:
                 if date == album[_PARTITIONS]:  # match the partition
                     value = album[name]
                     i = part_id[album[_PARTITIONS]]
                     part_id[album[_PARTITIONS]] += 1
                     if column.is_trie:
                         if column.rtrie_indicator == mdb.MDB_UINT_16:
                             val = rtrie.vid_for_value(vid16_nodes, vid16_kids, value)
                         else:
                             val = rtrie.vid_for_value(vid_nodes, vid_kids, value)
                     elif column.is_lz4:
                         val = clz4.compress(value)
                     else:
                         val = value
                     # self.assertEqual(db.get(txn, i), val)
                     if ind_db is not None:
                         #  row_id should be in bitmap too
                         if val in bitmaps:
                             bitmap = bitmaps[val]
                         else:
                             bitmap = BitSet()
                             bitmap.loads(ind_db.get(txn, val))
                             bitmaps[val] = bitmap
                         self.assertTrue(i in bitmap)
         txn.commit()
         env.close()
Esempio n. 5
0
def _convert_lz4(val, vid_trie=None, vid_trie16=None):
    try:
        val = str(val)
    except:
        val = val.encode('utf-8')
    return clz4.compress(val)
Esempio n. 6
0
def _convert_lz4(val, vid_trie=None, vid_trie16=None):
    try:
        val = str(val)
    except:
        val = val.encode('utf-8')
    return clz4.compress(val)