def test(self):
for condition in self.conditions:
c_usable_idxs = self.will_query_use_indexing(condition, {})
self.assertEqual(c_usable_idxs, self.usable_idxs,
"\nQuery with condition: ``%s``\n"
"Computed usable indexes are: ``%s``\n"
"and should be: ``%s``" %
(condition, c_usable_idxs, self.usable_idxs))
condvars = self.requiredExprVars(condition, None)
compiled = self.compileCondition(condition, condvars)
c_idx_expr = compiled.index_expressions
self.assertEqual(c_idx_expr, self.idx_expr,
"\nWrong index expression in condition:\n``%s``\n"
"Compiled index expression is:\n``%s``\n"
"and should be:\n``%s``" %
(condition, c_idx_expr, self.idx_expr))
c_str_expr = compiled.string_expression
self.assertEqual(c_str_expr, self.str_expr,
"\nWrong index operations in condition:\n``%s``\n"
"Computed index operations are:\n``%s``\n"
"and should be:\n``%s``" %
(condition, c_str_expr, self.str_expr))
vprint("* Query with condition ``%s`` will use "
"variables ``%s`` for indexing."
% (condition, compiled.index_variables))
def test(self):
for condition in self.conditions:
c_usable_idxs = self.will_query_use_indexing(condition, {})
self.assertEqual(c_usable_idxs, self.usable_idxs,
"\nQuery with condition: ``%s``\n"
"Computed usable indexes are: ``%s``\n"
"and should be: ``%s``" %
(condition, c_usable_idxs, self.usable_idxs))
condvars = self.requiredExprVars(condition, None)
compiled = self.compileCondition(condition, condvars)
c_idx_expr = compiled.index_expressions
self.assertEqual(c_idx_expr, self.idx_expr,
"\nWrong index expression in condition:\n``%s``\n"
"Compiled index expression is:\n``%s``\n"
"and should be:\n``%s``" %
(condition, c_idx_expr, self.idx_expr))
c_str_expr = compiled.string_expression
self.assertEqual(c_str_expr, self.str_expr,
"\nWrong index operations in condition:\n``%s``\n"
"Computed index operations are:\n``%s``\n"
"and should be:\n``%s``" %
(condition, c_str_expr, self.str_expr))
vprint("* Query with condition ``%s`` will use "
"variables ``%s`` for indexing."
% (condition, compiled.index_variables))
def createIndexes(self, colname, ncolname, extracolname):
if not self.indexed:
return
try:
kind = self.kind
vprint("* Indexing ``%s`` columns. Type: %s." % (colname, kind))
for acolname in [colname, ncolname, extracolname]:
acolumn = self.table.colinstances[acolname]
acolumn.create_index(
kind=self.kind, optlevel=self.optlevel,
_blocksizes=small_blocksizes, _testmode=True)
except TypeError, te:
if self.colNotIndexable_re.search(str(te)):
raise common.SkipTest(
"Columns of this type can not be indexed.")
raise
def createIndexes(self, colname, ncolname, extracolname):
if not self.indexed:
return
try:
kind = self.kind
vprint("* Indexing ``%s`` columns. Type: %s." % (colname, kind))
for acolname in [colname, ncolname, extracolname]:
acolumn = self.table.colinstances[acolname]
acolumn.createIndex(kind=self.kind,
optlevel=self.optlevel,
_blocksizes=small_blocksizes,
_testmode=True)
except TypeError, te:
if self.colNotIndexable_re.search(str(te)):
raise common.SkipTest(
"Columns of this type can not be indexed.")
raise
def create_indexes(self, colname, ncolname, extracolname):
if not self.indexed:
return
try:
kind = self.kind
vprint(f"* Indexing ``{colname}`` columns. Type: {kind}.")
for acolname in [colname, ncolname, extracolname]:
acolumn = self.table.colinstances[acolname]
acolumn.create_index(kind=self.kind,
optlevel=self.optlevel,
_blocksizes=small_blocksizes,
_testmode=True)
except TypeError as te:
if self.colNotIndexable_re.search(str(te)):
raise SilentlySkipTest(
"Columns of this type can not be indexed.")
raise
except NotImplementedError:
raise SilentlySkipTest(
"Indexing columns of this type is not supported yet.")
def test_method(self):
vprint("* Condition is ``%s``." % cond)
# Replace bitwise operators with their logical counterparts.
pycond = cond
for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
pycond = pycond.replace(ptop, pyop)
pycond = compile(pycond, '<string>', 'eval')
table = self.table
self.create_indexes(colname, ncolname, 'c_idxextra')
table_slice = dict(start=1, stop=table.nrows - 5, step=3)
rownos, fvalues = None, None
# Test that both simple and nested columns work as expected.
# Knowing how the table is filled, results must be the same.
for acolname in [colname, ncolname]:
# First the reference Python version.
pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
for row in table.iterrows(**table_slice):
pyvars[colname] = row[acolname]
pyvars['c_extra'] = row['c_extra']
pyvars['c_idxextra'] = row['c_idxextra']
try:
with warnings.catch_warnings():
warnings.filterwarnings(
'ignore',
'invalid value encountered in arc(cos|sin)',
RuntimeWarning)
isvalidrow = eval(pycond, func_info, pyvars)
except TypeError:
raise SilentlySkipTest(
"The Python type does not support the operation.")
if isvalidrow:
pyrownos.append(row.nrow)
pyfvalues.append(row[acolname])
pyrownos = numpy.array(pyrownos) # row numbers already sorted
pyfvalues = numpy.array(pyfvalues, dtype=sctype)
pyfvalues.sort()
vprint("* %d rows selected by Python from ``%s``."
% (len(pyrownos), acolname))
if rownos is None:
rownos = pyrownos # initialise reference results
fvalues = pyfvalues
else:
self.assertTrue(numpy.all(pyrownos == rownos)) # check
self.assertTrue(numpy.all(pyfvalues == fvalues))
# Then the in-kernel or indexed version.
ptvars = condvars.copy()
ptvars[colname] = table.colinstances[acolname]
ptvars['c_extra'] = table.colinstances['c_extra']
ptvars['c_idxextra'] = table.colinstances['c_idxextra']
try:
isidxq = table.will_query_use_indexing(cond, ptvars)
# Query twice to trigger possible query result caching.
ptrownos = [table.get_where_list(cond, condvars, sort=True,
**table_slice)
for _ in range(2)]
ptfvalues = [
table.read_where(cond, condvars, field=acolname,
**table_slice)
for _ in range(2)
]
except TypeError as te:
if self.condNotBoolean_re.search(str(te)):
raise SilentlySkipTest("The condition is not boolean.")
raise
except NotImplementedError:
raise SilentlySkipTest(
"The PyTables type does not support the operation.")
for ptfvals in ptfvalues: # row numbers already sorted
ptfvals.sort()
vprint("* %d rows selected by PyTables from ``%s``"
% (len(ptrownos[0]), acolname), nonl=True)
vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
self.assertTrue(numpy.all(ptrownos[0] == rownos))
self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
# The following test possible caching of query results.
self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
def test_method(self):
vprint("* Condition is ``%s``." % cond)
# Replace bitwise operators with their logical counterparts.
pycond = cond
for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
pycond = pycond.replace(ptop, pyop)
pycond = compile(pycond, '<string>', 'eval')
table = self.table
self.create_indexes(colname, ncolname, 'c_idxextra')
table_slice = dict(start=1, stop=table.nrows - 5, step=3)
rownos, fvalues = None, None
# Test that both simple and nested columns work as expected.
# Knowing how the table is filled, results must be the same.
for acolname in [colname, ncolname]:
# First the reference Python version.
pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
for row in table.iterrows(**table_slice):
pyvars[colname] = row[acolname]
pyvars['c_extra'] = row['c_extra']
pyvars['c_idxextra'] = row['c_idxextra']
try:
isvalidrow = eval(pycond, {}, pyvars)
except TypeError:
raise common.SkipTest(
"The Python type does not support the operation.")
if isvalidrow:
pyrownos.append(row.nrow)
pyfvalues.append(row[acolname])
pyrownos = numpy.array(pyrownos) # row numbers already sorted
pyfvalues = numpy.array(pyfvalues, dtype=sctype)
pyfvalues.sort()
vprint("* %d rows selected by Python from ``%s``."
% (len(pyrownos), acolname))
if rownos is None:
rownos = pyrownos # initialise reference results
fvalues = pyfvalues
else:
self.assertTrue(numpy.all(pyrownos == rownos)) # check
self.assertTrue(numpy.all(pyfvalues == fvalues))
# Then the in-kernel or indexed version.
ptvars = condvars.copy()
ptvars[colname] = table.colinstances[acolname]
ptvars['c_extra'] = table.colinstances['c_extra']
ptvars['c_idxextra'] = table.colinstances['c_idxextra']
try:
isidxq = table.will_query_use_indexing(cond, ptvars)
# Query twice to trigger possible query result caching.
ptrownos = [table.get_where_list(cond, condvars, sort=True,
**table_slice)
for _ in range(2)]
ptfvalues = [
table.read_where(cond, condvars, field=acolname,
**table_slice)
for _ in range(2)
]
except TypeError as te:
if self.condNotBoolean_re.search(str(te)):
raise common.SkipTest("The condition is not boolean.")
raise
except NotImplementedError:
raise common.SkipTest(
"The PyTables type does not support the operation.")
for ptfvals in ptfvalues: # row numbers already sorted
ptfvals.sort()
vprint("* %d rows selected by PyTables from ``%s``"
% (len(ptrownos[0]), acolname), nonl=True)
vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
self.assertTrue(numpy.all(ptrownos[0] == rownos))
self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
# The following test possible caching of query results.
self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
def create_test_method(type_, op, extracond):
sctype = sctype_from_type[type_]
# Compute the value of bounds.
condvars = {'bound': right_bound,
'lbound': left_bound,
'rbound': right_bound}
for (bname, bvalue) in condvars.iteritems():
if type_ == 'string':
bvalue = str_format % bvalue
bvalue = nxtype_from_type[type_](bvalue)
condvars[bname] = bvalue
# Compute the name of columns.
colname = 'c_%s' % type_
ncolname = 'c_nested/%s' % colname
# Compute the query condition.
if not op: # as is
cond = colname
elif op == '~': # unary
cond = '~(%s)' % colname
elif op == '<': # binary variable-constant
cond = '%s %s %s' % (colname, op, repr(condvars['bound']))
elif isinstance(op, tuple): # double binary variable-constant
cond = ('(lbound %s %s) & (%s %s rbound)'
% (op[0], colname, colname, op[1]))
else: # binary variable-variable
cond = '%s %s bound' % (colname, op)
if extracond:
cond = '(%s) %s' % (cond, extracond)
def test_method(self):
vprint("* Condition is ``%s``." % cond)
# Replace bitwise operators with their logical counterparts.
pycond = cond
for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
pycond = pycond.replace(ptop, pyop)
pycond = compile(pycond, '<string>', 'eval')
table = self.table
self.createIndexes(colname, ncolname, 'c_idxextra')
table_slice = dict(start=1, stop=table.nrows - 5, step=3)
rownos, fvalues = None, None
# Test that both simple and nested columns work as expected.
# Knowing how the table is filled, results must be the same.
for acolname in [colname, ncolname]:
# First the reference Python version.
pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
for row in table.iterrows(**table_slice):
pyvars[colname] = row[acolname]
pyvars['c_extra'] = row['c_extra']
pyvars['c_idxextra'] = row['c_idxextra']
try:
isvalidrow = eval(pycond, {}, pyvars)
except TypeError:
raise common.SkipTest(
"The Python type does not support the operation.")
if isvalidrow:
pyrownos.append(row.nrow)
pyfvalues.append(row[acolname])
pyrownos = numpy.array(pyrownos) # row numbers already sorted
pyfvalues = numpy.array(pyfvalues, dtype=sctype)
pyfvalues.sort()
vprint("* %d rows selected by Python from ``%s``."
% (len(pyrownos), acolname))
if rownos is None:
rownos = pyrownos # initialise reference results
fvalues = pyfvalues
else:
self.assertTrue(numpy.all(pyrownos == rownos)) # check
self.assertTrue(numpy.all(pyfvalues == fvalues))
# Then the in-kernel or indexed version.
ptvars = condvars.copy()
ptvars[colname] = table.colinstances[acolname]
ptvars['c_extra'] = table.colinstances['c_extra']
ptvars['c_idxextra'] = table.colinstances['c_idxextra']
try:
isidxq = table.will_query_use_indexing(cond, ptvars)
# Query twice to trigger possible query result caching.
ptrownos = [table.get_where_list(cond, condvars, sort=True,
**table_slice)
for _ in range(2)]
ptfvalues = [table.read_where(cond, condvars, field=acolname,
**table_slice)
for _ in range(2)]
except TypeError, te:
if self.condNotBoolean_re.search(str(te)):
raise common.SkipTest("The condition is not boolean.")
raise
except NotImplementedError:
raise common.SkipTest(
"The PyTables type does not support the operation.")
for ptfvals in ptfvalues: # row numbers already sorted
ptfvals.sort()
vprint("* %d rows selected by PyTables from ``%s``"
% (len(ptrownos[0]), acolname), nonl=True)
vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
self.assertTrue(numpy.all(ptrownos[0] == rownos))
self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
# The following test possible caching of query results.
self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
def create_test_method(type_, op, extracond):
sctype = sctype_from_type[type_]
# Compute the value of bounds.
condvars = {
'bound': right_bound,
'lbound': left_bound,
'rbound': right_bound
}
for (bname, bvalue) in condvars.iteritems():
if type_ == 'string':
bvalue = str_format % bvalue
bvalue = nxtype_from_type[type_](bvalue)
condvars[bname] = bvalue
# Compute the name of columns.
colname = 'c_%s' % type_
ncolname = 'c_nested/%s' % colname
# Compute the query condition.
if not op: # as is
cond = colname
elif op == '~': # unary
cond = '~(%s)' % colname
elif op == '<': # binary variable-constant
cond = '%s %s %s' % (colname, op, repr(condvars['bound']))
elif isinstance(op, tuple): # double binary variable-constant
cond = ('(lbound %s %s) & (%s %s rbound)' %
(op[0], colname, colname, op[1]))
else: # binary variable-variable
cond = '%s %s bound' % (colname, op)
if extracond:
cond = '(%s) %s' % (cond, extracond)
def test_method(self):
vprint("* Condition is ``%s``." % cond)
# Replace bitwise operators with their logical counterparts.
pycond = cond
for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
pycond = pycond.replace(ptop, pyop)
pycond = compile(pycond, '<string>', 'eval')
table = self.table
self.createIndexes(colname, ncolname, 'c_idxextra')
table_slice = dict(start=1, stop=table.nrows - 5, step=3)
rownos, fvalues = None, None
# Test that both simple and nested columns work as expected.
# Knowing how the table is filled, results must be the same.
for acolname in [colname, ncolname]:
# First the reference Python version.
pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
for row in table.iterrows(**table_slice):
pyvars[colname] = row[acolname]
pyvars['c_extra'] = row['c_extra']
pyvars['c_idxextra'] = row['c_idxextra']
try:
isvalidrow = eval(pycond, {}, pyvars)
except TypeError:
raise common.SkipTest(
"The Python type does not support the operation.")
if isvalidrow:
pyrownos.append(row.nrow)
pyfvalues.append(row[acolname])
pyrownos = numpy.array(pyrownos) # row numbers already sorted
pyfvalues = numpy.array(pyfvalues, dtype=sctype)
pyfvalues.sort()
vprint("* %d rows selected by Python from ``%s``." %
(len(pyrownos), acolname))
if rownos is None:
rownos = pyrownos # initialise reference results
fvalues = pyfvalues
else:
self.assertTrue(numpy.all(pyrownos == rownos)) # check
self.assertTrue(numpy.all(pyfvalues == fvalues))
# Then the in-kernel or indexed version.
ptvars = condvars.copy()
ptvars[colname] = table.colinstances[acolname]
ptvars['c_extra'] = table.colinstances['c_extra']
ptvars['c_idxextra'] = table.colinstances['c_idxextra']
try:
isidxq = table.willQueryUseIndexing(cond, ptvars)
# Query twice to trigger possible query result caching.
ptrownos = [
table.getWhereList(cond,
condvars,
sort=True,
**table_slice) for _ in range(2)
]
ptfvalues = [
table.readWhere(cond,
condvars,
field=acolname,
**table_slice) for _ in range(2)
]
except TypeError, te:
if self.condNotBoolean_re.search(str(te)):
raise common.SkipTest("The condition is not boolean.")
raise
except NotImplementedError:
raise common.SkipTest(
"The PyTables type does not support the operation.")
for ptfvals in ptfvalues: # row numbers already sorted
ptfvals.sort()
vprint("* %d rows selected by PyTables from ``%s``" %
(len(ptrownos[0]), acolname),
nonl=True)
vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
self.assertTrue(numpy.all(ptrownos[0] == rownos))
self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
# The following test possible caching of query results.
self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
