def setUp(self):
super(IndexIntegrityTest, self).setUp()
self.make_random_table()
# make some indexes
cols = [c for c in self._table.columns()][1:]
self._indexes = []
for c in cols:
name = c.get_name()
i = wt.Index(self._table, name)
i.add_key_column(c)
i.open("w")
i.build()
i.close()
self._indexes.append(i)
for c1, c2 in itertools.permutations(cols, 2):
name = c1.get_name() + "+" + c2.get_name()
i = wt.Index(self._table, name)
i.add_key_column(c1)
i.add_key_column(c2)
i.open("w")
i.build()
i.close()
self._indexes.append(i)
for c1, c2, c3 in itertools.permutations(cols, 3):
name = c1.get_name() + "+" + c2.get_name() + "+" + c3.get_name()
i = wt.Index(self._table, name)
i.add_key_column(c1)
i.add_key_column(c2)
i.add_key_column(c3)
i.open("w")
i.build()
i.close()
self._indexes.append(i)
def setUp(self):
super(BinnedIndexIntegrityTest, self).setUp()
self.make_random_table()
# make some indexes
cols = [c for c in self._table.columns()][1:]
self._indexes = []
for c in cols:
if c.get_type() in [wt.WT_INT, wt.WT_UINT]:
name = c.get_name()
for j in range(1, 10):
i = wt.Index(self._table, name + "_" + str(j))
i.add_key_column(c, j)
i.open("w")
i.build()
i.close()
self._indexes.append(i)
elif c.get_type() == wt.WT_FLOAT:
name = c.get_name()
w = 0.125
while w < 10:
i = wt.Index(self._table, name + "_" + str(w))
i.add_key_column(c, w)
i.open("w")
i.build()
i.close()
self._indexes.append(i)
w += 0.125
def test_max_prefix(self):
"""
Test the simplest possible case where we must get the maximum
key for a given prefix.
"""
t = wt.Table(self._homedir)
t.add_id_column(1)
t.add_char_column("s1")
t.add_char_column("s2")
t.open("w")
t.append([None, b"A", b"A"])
t.append([None, b"A", b"AA"])
t.append([None, b"A", b"B"])
t.append([None, b"AA", b""])
t.append([None, b"B", b"A"])
t.close()
t.open("r")
i = wt.Index(t, "test")
i.add_key_column(t.get_column("s1"))
i.add_key_column(t.get_column("s2"))
i.open("w")
i.build()
i.close()
i.open("r")
self.assertEqual(i.max_key(), (b"B", b"A"))
self.assertEqual(i.max_key("A"), (b"A", b"B"))
self.assertEqual(i.max_key("AA"), (b"AA", b""))
i.close()
t.close()
def test_unique_keys(self):
"""
Test the simplest possible case where we have keys that
cannot be distinguished if we concatentate the values
together.
"""
t = wt.Table(self._homedir)
t.add_id_column(1)
t.add_char_column("s1")
t.add_char_column("s2", num_elements=wt.WT_VAR_2)
t.open("w")
t.append([None, b"A", b"AA"])
t.append([None, b"AA", b"A"])
t.close()
t.open("r")
i = wt.Index(t, "test")
i.add_key_column(t.get_column("s1"))
i.add_key_column(t.get_column("s2"))
i.open("w")
i.build()
i.close()
i.open("r")
c = i.counter()
self.assertEqual(c[(b"A", b"AA")], 1)
self.assertEqual(c[(b"AA", b"A")], 1)
i.close()
t.close()
def test_dump_index(self):
cols = ["CHROM", "REF", "ALT"]
indexes = [wt.Index(self._table, col) for col in cols]
for i in indexes:
i.add_key_column(self._table.get_column(i.get_name()))
i.open("w")
i.build()
i.close()
i.open("r")
s = self.run_dump(["0", "--index=" + i.get_name()])
c = i.cursor([0])
n = 0
for line, r in zip(s.splitlines(), c):
n += 1
self.assertEqual(int(line), r[0])
self.assertEqual(n, len(self._table))
keys = [k for k in i.keys()]
for j in range(10):
k = random.randint(0, len(keys) - 1)
start = keys[k]
stop = keys[random.randint(k, len(keys) - 1)]
self.verify_dump(i, start, stop)
self.verify_dump(i, stop=stop)
self.verify_dump(i, start=start)
i.close()
i.delete()
def test_empty_prefix(self):
"""
Tests the simplest possible case where we have a
empty string prefix.
"""
# TODO we need to write a more general test class to exercise this
# code for min/max with prefixes more thoroughly.
t = wt.Table(self._homedir)
t.add_id_column(1)
t.add_char_column("s1")
t.add_char_column("s2")
t.add_char_column("s3")
t.open("w")
values = [(None, None, None), (None, None, b""), (None, None, b"z"),
(b'', b'', b'a'), (b'', b'', b'b'), (b'', b'', b'c'),
(b'', b'a', b'b'), (b'', b'a', b'z'), (b'b', b'b', b'b'),
(b'z', b'z', b'z')]
for r in values:
t.append([None] + list(r))
t.close()
t.open("r")
i = wt.Index(t, "test")
i.add_key_column(t.get_column("s1"))
i.add_key_column(t.get_column("s2"))
i.add_key_column(t.get_column("s3"))
i.open("w")
i.build()
i.close()
i.open("r")
# Max keys
self.assertEqual(i.max_key(None), values[2])
self.assertEqual(i.max_key(None, None), values[2])
self.assertEqual(i.max_key(None, None, b""), values[1])
self.assertEqual(i.max_key(b"", b""), values[5])
self.assertEqual(i.max_key(b"", b"", b"a"), values[3])
self.assertEqual(i.max_key(b"", b"", b"b"), values[4])
self.assertEqual(i.max_key(b"", b"", b"c"), values[5])
self.assertEqual(i.max_key(b"", b"a"), values[7])
self.assertEqual(i.max_key(b"", b"a", b"z"), values[7])
self.assertRaises(KeyError, i.max_key, "x")
self.assertRaises(KeyError, i.max_key, "", "x")
self.assertRaises(KeyError, i.max_key, None, "x")
# Min keys
self.assertEqual(i.min_key(None), values[0])
self.assertEqual(i.min_key(None, None), values[0])
self.assertEqual(i.min_key(None, None, b""), values[1])
self.assertEqual(i.min_key(b"", b""), values[3])
self.assertEqual(i.min_key(b"", b"", b"a"), values[3])
self.assertEqual(i.min_key(b"", b"", b"b"), values[4])
self.assertEqual(i.min_key(b"", b"", b"c"), values[5])
self.assertEqual(i.min_key(b"", b"a"), values[6])
self.assertEqual(i.min_key(b"", b"a", b"z"), values[7])
self.assertRaises(KeyError, i.min_key, "x")
self.assertRaises(KeyError, i.min_key, "", "x")
self.assertRaises(KeyError, i.min_key, None, "x")
i.close()
t.close()
def init(self):
super(AddRunner, self).init()
self._index = wt.Index(self._table, self._index_name)
if self._index.exists() and not self._force:
s = "Index '{0}' exists; use --force to overwrite"
self.error(s.format(self._index_name))
self.parse_colspec()
self._index.set_db_cache_size(self._index_db_cache_size)
self._index.open("w")
def test_add_index(self):
cols = ["CHROM", "REF", "ALT"]
for c in cols:
s = self.run_add([c, "-q"])
self.assertEqual(s, "")
i = wt.Index(self._table, c)
self.assertTrue(i.exists())
i.open("r")
self.assertEqual([col.get_name() for col in i.key_columns()], [c])
i.close()
def __init__(self, args):
super(DumpRunner, self).__init__(args)
self._db_cache_size = args.cache_size
self._index = None
self._start = args.start
self._stop = args.stop
self._columns = None
self._index_name = args.index
self._column_ids = args.columns
if args.index is not None:
self._index = wt.Index(self._table, self._index_name)
if not self._index.exists():
self.error("Index '{0}' not found".format(self._index_name))
def setUp(self):
super(MultivalueColumnTest, self).setUp()
# Add some indexes with specific properties.
index_cols = [["var1_uint"], ["var1_uint", "var1_int"],
["var1_uint", "var1_int", "var1_char"]]
index_cols = [[self._table.get_column(c) for c in cols]
for cols in index_cols]
for cols in index_cols:
name = "multi" + "+".join(c.get_name() for c in cols)
i = wt.Index(self._table, name)
for c in cols:
i.add_key_column(c)
i.open("w")
i.build()
i.close()
self._indexes.append(i)
def test_dump_multi_index(self):
i = wt.Index(self._table, "REF+ALT")
i.add_key_column(self._table.get_column("REF"))
i.add_key_column(self._table.get_column("ALT"))
i.open("w")
i.build()
i.close()
i.open("r")
s = self.run_dump(["0", "--index=" + i.get_name()])
c = i.cursor([0])
n = 0
for line, r in zip(s.splitlines(), c):
n += 1
self.assertEqual(int(line), r[0])
self.assertEqual(n, len(self._table))
self.verify_dump(i)
def test_permissions(self):
"""
Tests if the permissions on the DB and XML files are equal.
"""
t = self._table
i = wt.Index(t, "index")
i.add_key_column(t.get_column(1))
i.open("w")
i.build()
i.close()
for w in t, i:
# ensure the permissions are the same.
m1 = os.stat(w.get_metadata_path()).st_mode
m2 = os.stat(w.get_db_path()).st_mode
self.assertEqual(m1, m2)
m3 = os.stat(t.get_data_path()).st_mode
self.assertEqual(m1, m3)
def test_open(self):
name = "col1"
i = wt.Index(self._table, name)
self.assertEqual(i.get_db_name(), i.DB_PREFIX + name)
i.add_key_column(self._table.get_column(1))
self.assertFalse(i.is_open())
i.open("w")
self.assertTrue(i.is_open())
self.assertTrue(os.path.exists(i.get_db_build_path()))
i.build()
i.close()
self.assertFalse(os.path.exists(i.get_db_build_path()))
self.assertTrue(os.path.exists(i.get_metadata_path()))
self.assertTrue(os.path.exists(i.get_db_path()))
i.open("r")
keys = [k for k in i.keys()]
col = [r[1] for r in self._table]
self.assertEqual(keys, col)
i.close()
def test_open_api(self):
"""
Tests the open_table/index api to ensure everything works correctly.
"""
t = wt.Table(self._homedir)
t.add_id_column()
t.add_uint_column("u1")
t.open("w")
self.assertTrue(t.is_open())
t.close()
# open_table returns a table opened
self.assertFalse(t.is_open())
t = wt.open_table(self._homedir)
self.assertTrue(t.is_open())
t.close()
self.assertFalse(t.is_open())
# try now with the context manager
with wt.open_table(self._homedir) as t:
self.assertTrue(t.is_open())
self.assertFalse(t.is_open())
# Now do the same for an index.
t = wt.open_table(self._homedir)
name = "test"
i = wt.Index(t, name)
i.add_key_column(t.get_column(1))
i.open("w")
i.build()
i.close()
# The index is built, so we can open it.
i = t.open_index(name)
self.assertTrue(i.is_open())
i.close()
self.assertFalse(i.is_open())
with t.open_index(name) as i:
self.assertTrue(i.is_open())
self.assertFalse(i.is_open())
t.close()
def setUp(self):
super(IndexMinMaxTest, self).setUp()
self.set_values()
self.min_value = self.values[0]
self.max_value = self.values[-1]
t = wt.Table(self._homedir)
self._table = t
t.add_id_column(4)
for j in range(self.num_columns):
self.add_column("c_{0}".format(j))
t.open("w")
for a in ntuples(self.num_columns, len(self.values)):
v = [self.values[j] for j in a]
t.append([None] + v)
t.close()
t.open("r")
i = wt.Index(t, "test")
for j in range(self.num_columns):
i.add_key_column(t.get_column("c_{0}".format(j)))
i.open("w")
i.build()
i.close()
i.open("r")
self._index = i
def init(self):
super(IndexProgramRunner, self).init()
self._index = wt.Index(self._table, self._index_name)
if not self._index.exists():
self.error("Index '{0}' not found".format(self._index_name))
self._index.open("r")