Пример #1
0
 def test(self):
     for condition in self.conditions:
         c_usable_idxs = self.will_query_use_indexing(condition, {})
         self.assertEqual(c_usable_idxs, self.usable_idxs,
                          "\nQuery with condition: ``%s``\n"
                          "Computed usable indexes are: ``%s``\n"
                          "and should be: ``%s``" %
                         (condition, c_usable_idxs, self.usable_idxs))
         condvars = self.requiredExprVars(condition, None)
         compiled = self.compileCondition(condition, condvars)
         c_idx_expr = compiled.index_expressions
         self.assertEqual(c_idx_expr, self.idx_expr,
                          "\nWrong index expression in condition:\n``%s``\n"
                          "Compiled index expression is:\n``%s``\n"
                          "and should be:\n``%s``" %
                         (condition, c_idx_expr, self.idx_expr))
         c_str_expr = compiled.string_expression
         self.assertEqual(c_str_expr, self.str_expr,
                          "\nWrong index operations in condition:\n``%s``\n"
                          "Computed index operations are:\n``%s``\n"
                          "and should be:\n``%s``" %
                         (condition, c_str_expr, self.str_expr))
         vprint("* Query with condition ``%s`` will use "
                "variables ``%s`` for indexing."
                % (condition, compiled.index_variables))
Пример #2
0
 def test(self):
     for condition in self.conditions:
         c_usable_idxs = self.will_query_use_indexing(condition, {})
         self.assertEqual(c_usable_idxs, self.usable_idxs,
                          "\nQuery with condition: ``%s``\n"
                          "Computed usable indexes are: ``%s``\n"
                          "and should be: ``%s``" %
                         (condition, c_usable_idxs, self.usable_idxs))
         condvars = self.requiredExprVars(condition, None)
         compiled = self.compileCondition(condition, condvars)
         c_idx_expr = compiled.index_expressions
         self.assertEqual(c_idx_expr, self.idx_expr,
                          "\nWrong index expression in condition:\n``%s``\n"
                          "Compiled index expression is:\n``%s``\n"
                          "and should be:\n``%s``" %
                         (condition, c_idx_expr, self.idx_expr))
         c_str_expr = compiled.string_expression
         self.assertEqual(c_str_expr, self.str_expr,
                          "\nWrong index operations in condition:\n``%s``\n"
                          "Computed index operations are:\n``%s``\n"
                          "and should be:\n``%s``" %
                         (condition, c_str_expr, self.str_expr))
         vprint("* Query with condition ``%s`` will use "
                "variables ``%s`` for indexing."
                % (condition, compiled.index_variables))
Пример #3
0
    def createIndexes(self, colname, ncolname, extracolname):
        if not self.indexed:
            return
        try:
            kind = self.kind
            vprint("* Indexing ``%s`` columns. Type: %s." % (colname, kind))
            for acolname in [colname, ncolname, extracolname]:
                acolumn = self.table.colinstances[acolname]
                acolumn.create_index(
                    kind=self.kind, optlevel=self.optlevel,
                    _blocksizes=small_blocksizes, _testmode=True)

        except TypeError, te:
            if self.colNotIndexable_re.search(str(te)):
                raise common.SkipTest(
                    "Columns of this type can not be indexed.")
            raise
Пример #4
0
    def createIndexes(self, colname, ncolname, extracolname):
        if not self.indexed:
            return
        try:
            kind = self.kind
            vprint("* Indexing ``%s`` columns. Type: %s." % (colname, kind))
            for acolname in [colname, ncolname, extracolname]:
                acolumn = self.table.colinstances[acolname]
                acolumn.createIndex(kind=self.kind,
                                    optlevel=self.optlevel,
                                    _blocksizes=small_blocksizes,
                                    _testmode=True)

        except TypeError, te:
            if self.colNotIndexable_re.search(str(te)):
                raise common.SkipTest(
                    "Columns of this type can not be indexed.")
            raise
Пример #5
0
    def create_indexes(self, colname, ncolname, extracolname):
        if not self.indexed:
            return
        try:
            kind = self.kind
            vprint(f"* Indexing ``{colname}`` columns. Type: {kind}.")
            for acolname in [colname, ncolname, extracolname]:
                acolumn = self.table.colinstances[acolname]
                acolumn.create_index(kind=self.kind,
                                     optlevel=self.optlevel,
                                     _blocksizes=small_blocksizes,
                                     _testmode=True)

        except TypeError as te:
            if self.colNotIndexable_re.search(str(te)):
                raise SilentlySkipTest(
                    "Columns of this type can not be indexed.")
            raise
        except NotImplementedError:
            raise SilentlySkipTest(
                "Indexing columns of this type is not supported yet.")
Пример #6
0
    def test_method(self):
        vprint("* Condition is ``%s``." % cond)
        # Replace bitwise operators with their logical counterparts.
        pycond = cond
        for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
            pycond = pycond.replace(ptop, pyop)
        pycond = compile(pycond, '<string>', 'eval')

        table = self.table
        self.create_indexes(colname, ncolname, 'c_idxextra')

        table_slice = dict(start=1, stop=table.nrows - 5, step=3)
        rownos, fvalues = None, None
        # Test that both simple and nested columns work as expected.
        # Knowing how the table is filled, results must be the same.
        for acolname in [colname, ncolname]:
            # First the reference Python version.
            pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
            for row in table.iterrows(**table_slice):
                pyvars[colname] = row[acolname]
                pyvars['c_extra'] = row['c_extra']
                pyvars['c_idxextra'] = row['c_idxextra']
                try:
                    with warnings.catch_warnings():
                        warnings.filterwarnings(
                            'ignore',
                            'invalid value encountered in arc(cos|sin)',
                            RuntimeWarning)
                        isvalidrow = eval(pycond, func_info, pyvars)
                except TypeError:
                    raise SilentlySkipTest(
                        "The Python type does not support the operation.")
                if isvalidrow:
                    pyrownos.append(row.nrow)
                    pyfvalues.append(row[acolname])
            pyrownos = numpy.array(pyrownos)  # row numbers already sorted
            pyfvalues = numpy.array(pyfvalues, dtype=sctype)
            pyfvalues.sort()
            vprint("* %d rows selected by Python from ``%s``."
                   % (len(pyrownos), acolname))
            if rownos is None:
                rownos = pyrownos  # initialise reference results
                fvalues = pyfvalues
            else:
                self.assertTrue(numpy.all(pyrownos == rownos))  # check
                self.assertTrue(numpy.all(pyfvalues == fvalues))

            # Then the in-kernel or indexed version.
            ptvars = condvars.copy()
            ptvars[colname] = table.colinstances[acolname]
            ptvars['c_extra'] = table.colinstances['c_extra']
            ptvars['c_idxextra'] = table.colinstances['c_idxextra']
            try:
                isidxq = table.will_query_use_indexing(cond, ptvars)
                # Query twice to trigger possible query result caching.
                ptrownos = [table.get_where_list(cond, condvars, sort=True,
                                                 **table_slice)
                            for _ in range(2)]
                ptfvalues = [
                    table.read_where(cond, condvars, field=acolname,
                                     **table_slice)
                    for _ in range(2)
                ]
            except TypeError as te:
                if self.condNotBoolean_re.search(str(te)):
                    raise SilentlySkipTest("The condition is not boolean.")
                raise
            except NotImplementedError:
                raise SilentlySkipTest(
                    "The PyTables type does not support the operation.")
            for ptfvals in ptfvalues:  # row numbers already sorted
                ptfvals.sort()
            vprint("* %d rows selected by PyTables from ``%s``"
                   % (len(ptrownos[0]), acolname), nonl=True)
            vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
            self.assertTrue(numpy.all(ptrownos[0] == rownos))
            self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
            # The following test possible caching of query results.
            self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
            self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
Пример #7
0
    def test_method(self):
        vprint("* Condition is ``%s``." % cond)
        # Replace bitwise operators with their logical counterparts.
        pycond = cond
        for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
            pycond = pycond.replace(ptop, pyop)
        pycond = compile(pycond, '<string>', 'eval')

        table = self.table
        self.create_indexes(colname, ncolname, 'c_idxextra')

        table_slice = dict(start=1, stop=table.nrows - 5, step=3)
        rownos, fvalues = None, None
        # Test that both simple and nested columns work as expected.
        # Knowing how the table is filled, results must be the same.
        for acolname in [colname, ncolname]:
            # First the reference Python version.
            pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
            for row in table.iterrows(**table_slice):
                pyvars[colname] = row[acolname]
                pyvars['c_extra'] = row['c_extra']
                pyvars['c_idxextra'] = row['c_idxextra']
                try:
                    isvalidrow = eval(pycond, {}, pyvars)
                except TypeError:
                    raise common.SkipTest(
                        "The Python type does not support the operation.")
                if isvalidrow:
                    pyrownos.append(row.nrow)
                    pyfvalues.append(row[acolname])
            pyrownos = numpy.array(pyrownos)  # row numbers already sorted
            pyfvalues = numpy.array(pyfvalues, dtype=sctype)
            pyfvalues.sort()
            vprint("* %d rows selected by Python from ``%s``."
                   % (len(pyrownos), acolname))
            if rownos is None:
                rownos = pyrownos  # initialise reference results
                fvalues = pyfvalues
            else:
                self.assertTrue(numpy.all(pyrownos == rownos))  # check
                self.assertTrue(numpy.all(pyfvalues == fvalues))

            # Then the in-kernel or indexed version.
            ptvars = condvars.copy()
            ptvars[colname] = table.colinstances[acolname]
            ptvars['c_extra'] = table.colinstances['c_extra']
            ptvars['c_idxextra'] = table.colinstances['c_idxextra']
            try:
                isidxq = table.will_query_use_indexing(cond, ptvars)
                # Query twice to trigger possible query result caching.
                ptrownos = [table.get_where_list(cond, condvars, sort=True,
                                                 **table_slice)
                            for _ in range(2)]
                ptfvalues = [
                    table.read_where(cond, condvars, field=acolname,
                                     **table_slice)
                    for _ in range(2)
                ]
            except TypeError as te:
                if self.condNotBoolean_re.search(str(te)):
                    raise common.SkipTest("The condition is not boolean.")
                raise
            except NotImplementedError:
                raise common.SkipTest(
                    "The PyTables type does not support the operation.")
            for ptfvals in ptfvalues:  # row numbers already sorted
                ptfvals.sort()
            vprint("* %d rows selected by PyTables from ``%s``"
                   % (len(ptrownos[0]), acolname), nonl=True)
            vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
            self.assertTrue(numpy.all(ptrownos[0] == rownos))
            self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
            # The following test possible caching of query results.
            self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
            self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
Пример #8
0
def create_test_method(type_, op, extracond):
    sctype = sctype_from_type[type_]

    # Compute the value of bounds.
    condvars = {'bound': right_bound,
                'lbound': left_bound,
                'rbound': right_bound}
    for (bname, bvalue) in condvars.iteritems():
        if type_ == 'string':
            bvalue = str_format % bvalue
        bvalue = nxtype_from_type[type_](bvalue)
        condvars[bname] = bvalue

    # Compute the name of columns.
    colname = 'c_%s' % type_
    ncolname = 'c_nested/%s' % colname

    # Compute the query condition.
    if not op:  # as is
        cond = colname
    elif op == '~':  # unary
        cond = '~(%s)' % colname
    elif op == '<':  # binary variable-constant
        cond = '%s %s %s' % (colname, op, repr(condvars['bound']))
    elif isinstance(op, tuple):  # double binary variable-constant
        cond = ('(lbound %s %s) & (%s %s rbound)'
                % (op[0], colname, colname, op[1]))
    else:  # binary variable-variable
        cond = '%s %s bound' % (colname, op)
    if extracond:
        cond = '(%s) %s' % (cond, extracond)

    def test_method(self):
        vprint("* Condition is ``%s``." % cond)
        # Replace bitwise operators with their logical counterparts.
        pycond = cond
        for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
            pycond = pycond.replace(ptop, pyop)
        pycond = compile(pycond, '<string>', 'eval')

        table = self.table
        self.createIndexes(colname, ncolname, 'c_idxextra')

        table_slice = dict(start=1, stop=table.nrows - 5, step=3)
        rownos, fvalues = None, None
        # Test that both simple and nested columns work as expected.
        # Knowing how the table is filled, results must be the same.
        for acolname in [colname, ncolname]:
            # First the reference Python version.
            pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
            for row in table.iterrows(**table_slice):
                pyvars[colname] = row[acolname]
                pyvars['c_extra'] = row['c_extra']
                pyvars['c_idxextra'] = row['c_idxextra']
                try:
                    isvalidrow = eval(pycond, {}, pyvars)
                except TypeError:
                    raise common.SkipTest(
                        "The Python type does not support the operation.")
                if isvalidrow:
                    pyrownos.append(row.nrow)
                    pyfvalues.append(row[acolname])
            pyrownos = numpy.array(pyrownos)  # row numbers already sorted
            pyfvalues = numpy.array(pyfvalues, dtype=sctype)
            pyfvalues.sort()
            vprint("* %d rows selected by Python from ``%s``."
                   % (len(pyrownos), acolname))
            if rownos is None:
                rownos = pyrownos  # initialise reference results
                fvalues = pyfvalues
            else:
                self.assertTrue(numpy.all(pyrownos == rownos))  # check
                self.assertTrue(numpy.all(pyfvalues == fvalues))

            # Then the in-kernel or indexed version.
            ptvars = condvars.copy()
            ptvars[colname] = table.colinstances[acolname]
            ptvars['c_extra'] = table.colinstances['c_extra']
            ptvars['c_idxextra'] = table.colinstances['c_idxextra']
            try:
                isidxq = table.will_query_use_indexing(cond, ptvars)
                # Query twice to trigger possible query result caching.
                ptrownos = [table.get_where_list(cond, condvars, sort=True,
                                                 **table_slice)
                            for _ in range(2)]
                ptfvalues = [table.read_where(cond, condvars, field=acolname,
                                              **table_slice)
                                             for _ in range(2)]
            except TypeError, te:
                if self.condNotBoolean_re.search(str(te)):
                    raise common.SkipTest("The condition is not boolean.")
                raise
            except NotImplementedError:
                raise common.SkipTest(
                    "The PyTables type does not support the operation.")
            for ptfvals in ptfvalues:  # row numbers already sorted
                ptfvals.sort()
            vprint("* %d rows selected by PyTables from ``%s``"
                   % (len(ptrownos[0]), acolname), nonl=True)
            vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
            self.assertTrue(numpy.all(ptrownos[0] == rownos))
            self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
            # The following test possible caching of query results.
            self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
            self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))
Пример #9
0
def create_test_method(type_, op, extracond):
    sctype = sctype_from_type[type_]

    # Compute the value of bounds.
    condvars = {
        'bound': right_bound,
        'lbound': left_bound,
        'rbound': right_bound
    }
    for (bname, bvalue) in condvars.iteritems():
        if type_ == 'string':
            bvalue = str_format % bvalue
        bvalue = nxtype_from_type[type_](bvalue)
        condvars[bname] = bvalue

    # Compute the name of columns.
    colname = 'c_%s' % type_
    ncolname = 'c_nested/%s' % colname

    # Compute the query condition.
    if not op:  # as is
        cond = colname
    elif op == '~':  # unary
        cond = '~(%s)' % colname
    elif op == '<':  # binary variable-constant
        cond = '%s %s %s' % (colname, op, repr(condvars['bound']))
    elif isinstance(op, tuple):  # double binary variable-constant
        cond = ('(lbound %s %s) & (%s %s rbound)' %
                (op[0], colname, colname, op[1]))
    else:  # binary variable-variable
        cond = '%s %s bound' % (colname, op)
    if extracond:
        cond = '(%s) %s' % (cond, extracond)

    def test_method(self):
        vprint("* Condition is ``%s``." % cond)
        # Replace bitwise operators with their logical counterparts.
        pycond = cond
        for (ptop, pyop) in [('&', 'and'), ('|', 'or'), ('~', 'not')]:
            pycond = pycond.replace(ptop, pyop)
        pycond = compile(pycond, '<string>', 'eval')

        table = self.table
        self.createIndexes(colname, ncolname, 'c_idxextra')

        table_slice = dict(start=1, stop=table.nrows - 5, step=3)
        rownos, fvalues = None, None
        # Test that both simple and nested columns work as expected.
        # Knowing how the table is filled, results must be the same.
        for acolname in [colname, ncolname]:
            # First the reference Python version.
            pyrownos, pyfvalues, pyvars = [], [], condvars.copy()
            for row in table.iterrows(**table_slice):
                pyvars[colname] = row[acolname]
                pyvars['c_extra'] = row['c_extra']
                pyvars['c_idxextra'] = row['c_idxextra']
                try:
                    isvalidrow = eval(pycond, {}, pyvars)
                except TypeError:
                    raise common.SkipTest(
                        "The Python type does not support the operation.")
                if isvalidrow:
                    pyrownos.append(row.nrow)
                    pyfvalues.append(row[acolname])
            pyrownos = numpy.array(pyrownos)  # row numbers already sorted
            pyfvalues = numpy.array(pyfvalues, dtype=sctype)
            pyfvalues.sort()
            vprint("* %d rows selected by Python from ``%s``." %
                   (len(pyrownos), acolname))
            if rownos is None:
                rownos = pyrownos  # initialise reference results
                fvalues = pyfvalues
            else:
                self.assertTrue(numpy.all(pyrownos == rownos))  # check
                self.assertTrue(numpy.all(pyfvalues == fvalues))

            # Then the in-kernel or indexed version.
            ptvars = condvars.copy()
            ptvars[colname] = table.colinstances[acolname]
            ptvars['c_extra'] = table.colinstances['c_extra']
            ptvars['c_idxextra'] = table.colinstances['c_idxextra']
            try:
                isidxq = table.willQueryUseIndexing(cond, ptvars)
                # Query twice to trigger possible query result caching.
                ptrownos = [
                    table.getWhereList(cond,
                                       condvars,
                                       sort=True,
                                       **table_slice) for _ in range(2)
                ]
                ptfvalues = [
                    table.readWhere(cond,
                                    condvars,
                                    field=acolname,
                                    **table_slice) for _ in range(2)
                ]
            except TypeError, te:
                if self.condNotBoolean_re.search(str(te)):
                    raise common.SkipTest("The condition is not boolean.")
                raise
            except NotImplementedError:
                raise common.SkipTest(
                    "The PyTables type does not support the operation.")
            for ptfvals in ptfvalues:  # row numbers already sorted
                ptfvals.sort()
            vprint("* %d rows selected by PyTables from ``%s``" %
                   (len(ptrownos[0]), acolname),
                   nonl=True)
            vprint("(indexing: %s)." % ["no", "yes"][bool(isidxq)])
            self.assertTrue(numpy.all(ptrownos[0] == rownos))
            self.assertTrue(numpy.all(ptfvalues[0] == fvalues))
            # The following test possible caching of query results.
            self.assertTrue(numpy.all(ptrownos[0] == ptrownos[1]))
            self.assertTrue(numpy.all(ptfvalues[0] == ptfvalues[1]))