def assert_missing_op(self, op, expr, local_dict):
     msg = "expected NotImplementedError regarding '%s'" % op
     try:
         evaluate(expr, local_dict)
     except NotImplementedError, nie:
         if "'%s'" % op not in nie.args[0]:
             self.fail(msg)
示例#2
0
 def assert_missing_op(self, op, expr, local_dict):
     msg = "expected NotImplementedError regarding '%s'" % op
     try:
         evaluate(expr, local_dict)
     except NotImplementedError, nie:
         if "'%s'" % op not in nie.args[0]:
             self.fail(msg)
示例#3
0
 def test_illegal_value(self):
     a = arange(3)
     try:
         evaluate("a < [0, 0, 0]")
     except TypeError:
         pass
     else:
         self.fail()
示例#4
0
 def test_broadcasting(self):
     a = arange(100).reshape(10, 10)[::2]
     c = arange(10)
     d = arange(5).reshape(5, 1)
     assert_array_equal(evaluate("a+c"), a + c)
     assert_array_equal(evaluate("a+d"), a + d)
     expr = numexpr("2.0*a+3.0*c", [('a', float), ('c', float)])
     assert_array_equal(expr(a, c), 2.0 * a + 3.0 * c)
 def test_broadcasting(self):
     a = arange(100).reshape(10,10)[::2]
     c = arange(10)
     d = arange(5).reshape(5,1)
     assert_array_equal(evaluate("a+c"), a+c)
     assert_array_equal(evaluate("a+d"), a+d)
     expr = numexpr("2.0*a+3.0*c",[('a',float),('c', float)])
     assert_array_equal(expr(a,c), 2.0*a+3.0*c)
 def test_illegal_value(self):
     a = arange(3)
     try:
         evaluate("a < [0, 0, 0]")
     except TypeError:
         pass
     else:
         self.fail()
 def test_long_constant_promotion(self):
     int32array = arange(100, dtype='int32')
     res = int32array * 2
     res32 = evaluate('int32array * 2')
     res64 = evaluate('int32array * 2L')
     assert_array_equal(res, res32)
     assert_array_equal(res, res64)
     self.assertEqual(res32.dtype.name, 'int32')
     self.assertEqual(res64.dtype.name, 'int64')
示例#8
0
 def test_long_constant_promotion(self):
     int32array = arange(100, dtype='int32')
     res = int32array * 2
     res32 = evaluate('int32array * 2')
     res64 = evaluate('int32array * 2L')
     assert_array_equal(res, res32)
     assert_array_equal(res, res64)
     self.assertEqual(res32.dtype.name, 'int32')
     self.assertEqual(res64.dtype.name, 'int64')
 def test_complex_strides(self):
     a = arange(100).reshape(10,10)[::2]
     b = arange(50).reshape(5,10)
     assert_array_equal(evaluate("a+b"), a+b)
     c = empty([10], dtype=[('c1', int32), ('c2', uint16)])
     c['c1'] = arange(10)
     c['c2'].fill(0xaaaa)
     c1 = c['c1']
     a0 = a[0]
     assert_array_equal(evaluate("c1"), c1)
     assert_array_equal(evaluate("a0+c1"), a0+c1)
示例#10
0
 def test_complex_strides(self):
     a = arange(100).reshape(10, 10)[::2]
     b = arange(50).reshape(5, 10)
     assert_array_equal(evaluate("a+b"), a + b)
     c = empty([10], dtype=[('c1', int32), ('c2', uint16)])
     c['c1'] = arange(10)
     c['c2'].fill(0xaaaa)
     c1 = c['c1']
     a0 = a[0]
     assert_array_equal(evaluate("c1"), c1)
     assert_array_equal(evaluate("a0+c1"), a0 + c1)
示例#11
0
 def test_axis(self):
     y = arange(9.0).reshape(3, 3)
     try:
         evaluate("sum(y, axis=2)")
     except ValueError:
         pass
     else:
         raise ValueError("should raise exception!")
     try:
         evaluate("sum(y, axis=-3)")
     except ValueError:
         pass
     else:
         raise ValueError("should raise exception!")
示例#12
0
    def test_select(self):
        f0 = arange(10, dtype=int32)
        f1 = arange(10, dtype=float64)

        irregular = rec.fromarrays([f0, f1])

        f0 = irregular['f0']
        f1 = irregular['f1']

        i0 = evaluate('f0 < 5')
        i1 = evaluate('f1 < 5')

        assert_array_equal(f0[i0], arange(5, dtype=int32))
        assert_array_equal(f1[i1], arange(5, dtype=float64))
示例#13
0
    def test_select(self):
        f0 = arange(10, dtype=int32)
        f1 = arange(10, dtype=float64)

        irregular = rec.fromarrays([f0, f1])

        f0 = irregular['f0']
        f1 = irregular['f1']

        i0 = evaluate('f0 < 5')
        i1 = evaluate('f1 < 5')

        assert_array_equal(f0[i0], arange(5, dtype=int32))
        assert_array_equal(f1[i1], arange(5, dtype=float64))
示例#14
0
 def test_axis(self):
     y = arange(9.0).reshape(3,3)
     try:
         evaluate("sum(y, axis=2)")
     except ValueError:
         pass
     else:
         raise ValueError("should raise exception!")
     try:
         evaluate("sum(y, axis=-3)")
     except ValueError:
         pass
     else:
         raise ValueError("should raise exception!")
示例#15
0
 def test_int64_array_promotion(self):
     int32array = arange(100, dtype='int32')
     int64array = arange(100, dtype='int64')
     respy = int32array * int64array
     resnx = evaluate('int32array * int64array')
     assert_array_equal(respy, resnx)
     self.assertEqual(resnx.dtype.name, 'int64')
示例#16
0
 def test_null_chars(self):
     str_list = [
         '\0\0\0', '\0\0foo\0', '\0\0foo\0b', '\0\0foo\0b\0',
         'foo\0', 'foo\0b', 'foo\0b\0', 'foo\0bar\0baz\0\0' ]
     for s in str_list:
         r = evaluate('s')
         self.assertEqual(s, r.tostring())  # check *all* stored data
示例#17
0
 def test_compare_array(self):
     sarr1 = self.str_array1
     sarr2 = self.str_array2
     expr = 'sarr1 >= sarr2'
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#18
0
 def test_compare_variable(self):
     sarr = self.str_array1
     svar = self.str_constant
     expr = 'sarr >= svar'
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#19
0
 def test_compare_array(self):
     sarr1 = self.str_array1
     sarr2 = self.str_array2
     expr = 'sarr1 >= sarr2'
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#20
0
 def test_compare_variable(self):
     sarr = self.str_array1
     svar = self.str_constant
     expr = 'sarr >= svar'
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#21
0
 def test_int64_array_promotion(self):
     int32array = arange(100, dtype='int32')
     int64array = arange(100, dtype='int64')
     respy = int32array * int64array
     resnx = evaluate('int32array * int64array')
     assert_array_equal(respy, resnx)
     self.assertEqual(resnx.dtype.name, 'int64')
示例#22
0
 def test_null_chars(self):
     str_list = [
         '\0\0\0', '\0\0foo\0', '\0\0foo\0b', '\0\0foo\0b\0', 'foo\0',
         'foo\0b', 'foo\0b\0', 'foo\0bar\0baz\0\0'
     ]
     for s in str_list:
         r = evaluate('s')
         self.assertEqual(s, r.tostring())  # check *all* stored data
示例#23
0
 def test_complex_expr(self):
     def complex(a, b):
         c = zeros(a.shape, dtype=complex_)
         c.real = a
         c.imag = b
         return c
     a = arange(1e4)
     b = arange(1e4)**1e-5
     z = a + 1j*b
     x = z.imag
     x = sin(complex(a, b)).real + z.imag
     y = evaluate("sin(complex(a, b)).real + z.imag")
     assert_array_almost_equal(x, y)
示例#24
0
    def test_complex_expr(self):
        def complex(a, b):
            c = zeros(a.shape, dtype=complex_)
            c.real = a
            c.imag = b
            return c

        a = arange(1e4)
        b = arange(1e4)**1e-5
        z = a + 1j * b
        x = z.imag
        x = sin(complex(a, b)).real + z.imag
        y = evaluate("sin(complex(a, b)).real + z.imag")
        assert_array_almost_equal(x, y)
示例#25
0
        def method(self):
            npval = eval(expr, globals(), this_locals)
            try:
                neval = evaluate(expr, local_dict=this_locals,
                                 optimization=optimization)
                assert equal(npval, neval, exact), \
                    """%r
(test_scalar=%r, dtype=%r, optimization=%r, exact=%r,
 npval=%r (%r), neval=%r (%r))""" % (expr, test_scalar, dtype.__name__,
                                     optimization, exact,
                                     npval, type(npval), neval, type(neval))
            except AssertionError:
                raise
            except NotImplementedError:
                print('%r not implemented for %s' % (expr,dtype.__name__))
            except:
                print('numexpr error for expression %r' % (expr,))
                raise
示例#26
0
        def method(self):
            npval = eval(expr, globals(), this_locals)
            try:
                neval = evaluate(expr,
                                 local_dict=this_locals,
                                 optimization=optimization)
                assert equal(npval, neval, exact), \
                    """%r
(test_scalar=%r, dtype=%r, optimization=%r, exact=%r,
 npval=%r (%r), neval=%r (%r))""" % (expr, test_scalar, dtype.__name__,
                                     optimization, exact,
                                     npval, type(npval), neval, type(neval))
            except AssertionError:
                raise
            except NotImplementedError:
                print('%r not implemented for %s' % (expr, dtype.__name__))
            except:
                print('numexpr error for expression %r' % (expr, ))
                raise
示例#27
0
    def test_compare_prefix(self):
        # Check comparing two strings where one is a prefix of the
        # other.
        for s1, s2 in [ ('foo', 'foobar'), ('foo', 'foo\0bar'),
                        ('foo\0a', 'foo\0bar') ]:
            self.assert_(evaluate('s1 < s2'))
            self.assert_(evaluate('s1 <= s2'))
            self.assert_(evaluate('~(s1 == s2)'))
            self.assert_(evaluate('~(s1 >= s2)'))
            self.assert_(evaluate('~(s1 > s2)'))

        # Check for NumPy array-style semantics in string equality.
        s1, s2 = 'foo', 'foo\0\0'
        self.assert_(evaluate('s1 == s2'))
示例#28
0
    def test_compare_prefix(self):
        # Check comparing two strings where one is a prefix of the
        # other.
        for s1, s2 in [('foo', 'foobar'), ('foo', 'foo\0bar'),
                       ('foo\0a', 'foo\0bar')]:
            self.assert_(evaluate('s1 < s2'))
            self.assert_(evaluate('s1 <= s2'))
            self.assert_(evaluate('~(s1 == s2)'))
            self.assert_(evaluate('~(s1 >= s2)'))
            self.assert_(evaluate('~(s1 > s2)'))

        # Check for NumPy array-style semantics in string equality.
        s1, s2 = 'foo', 'foo\0\0'
        self.assert_(evaluate('s1 == s2'))
示例#29
0
 def run(self):
     a = arange(3)
     assert_array_equal(evaluate('a**3'), array([0, 1, 8]))
示例#30
0
 def test_big_int(self):
     # Big ints should be promoted to longs.
     # This test may only fail under 64-bit platforms.
     res = evaluate('2**40')
     assert_array_equal(res, 2**40)
     self.assertEqual(res.dtype.name, 'int64')
示例#31
0
 def test_simple_expr(self):
     x = arange(1e5)
     y = evaluate("x")
     assert_array_equal(x, y)
示例#32
0
 def test_simple_expr_small_array(self):
     x = arange(100.0)
     y = evaluate("x")
     assert_array_equal(x, y)
示例#33
0
 def test_simple_expr_small_array(self):
     x = arange(100.0)
     y = evaluate("x")
     assert_array_equal(x, y)
示例#34
0
 def test_all_scalar(self):
     a = 3.
     b = 4.
     assert_equal(evaluate("a+b"), a+b)
     expr = numexpr("2*a+3*b",[('a',float),('b', float)])
     assert_equal(expr(a,b), 2*a+3*b)
示例#35
0
 def test_compare_copy(self):
     sarr = self.str_array1
     expr = 'sarr'
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#36
0
 def test_all_scalar(self):
     a = 3.
     b = 4.
     assert_equal(evaluate("a+b"), a + b)
     expr = numexpr("2*a+3*b", [('a', float), ('b', float)])
     assert_equal(expr(a, b), 2 * a + 3 * b)
示例#37
0
 def test_simple(self):
     a = array([1., 2., 3.])
     b = array([4., 5., 6.])
     c = array([7., 8., 9.])
     x = evaluate("2*a + 3*b*c")
     assert_array_equal(x, array([86., 124., 168.]))
示例#38
0
 def test_small_long(self):
     # Small longs should not be downgraded to ints.
     res = evaluate('42L')
     assert_array_equal(res, 42)
     self.assertEqual(res.dtype.name, 'int64')
示例#39
0
 def test_rational_expr(self):
     a = arange(1e5)
     b = arange(1e5) * 0.1
     x = (a + 2*b) / (1 + a + 4*b*b)
     y = evaluate("(a + 2*b) / (1 + a + 4*b*b)")
     assert_array_equal(x, y)
示例#40
0
 def test_simple_expr(self):
     x = arange(1e5)
     y = evaluate("x")
     assert_array_equal(x, y)
示例#41
0
 def test_compare_copy(self):
     sarr = self.str_array1
     expr = 'sarr'
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#42
0
 def test_reductions(self):
     # Check that they compile OK.
     assert_equal(disassemble(
         numexpr("sum(x**2+2, axis=None)", [('x', float)])),
                  [('mul_fff', 't3', 'r1[x]', 'r1[x]'),
                   ('add_fff', 't3', 't3', 'c2[2.0]'),
                   ('sum_ffn', 'r0', 't3', None)])
     assert_equal(disassemble(
         numexpr("sum(x**2+2, axis=1)", [('x', float)])),
                  [('mul_fff', 't3', 'r1[x]', 'r1[x]'),
                   ('add_fff', 't3', 't3', 'c2[2.0]'),
                   ('sum_ffn', 'r0', 't3', 1)])
     assert_equal(disassemble(
         numexpr("prod(x**2+2, axis=2)", [('x', float)])),
                  [('mul_fff', 't3', 'r1[x]', 'r1[x]'),
                   ('add_fff', 't3', 't3', 'c2[2.0]'),
                   ('prod_ffn', 'r0', 't3', 2)])
     # Check that full reductions work.
     x = arange(10.0)
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0))
     # Check that reductions along an axis work
     y = arange(9.0).reshape(3,3)
     assert_equal(evaluate("sum(y**2, axis=1)"), sum(y**2, axis=1))
     assert_equal(evaluate("sum(y**2, axis=0)"), sum(y**2, axis=0))
     assert_equal(evaluate("sum(y**2, axis=None)"), sum(y**2, axis=None))
     assert_equal(evaluate("prod(y**2, axis=1)"), prod(y**2, axis=1))
     assert_equal(evaluate("prod(y**2, axis=0)"), prod(y**2, axis=0))
     assert_equal(evaluate("prod(y**2, axis=None)"), prod(y**2, axis=None))
     # Check integers
     x = x.astype(int)
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0))
     # Check longs
     x = x.astype(long)
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0))
     # Check complex
     x = x + 5j
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0))
示例#43
0
 def test_compare_constant(self):
     sarr = self.str_array1
     expr = 'sarr >= %r' % self.str_constant
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#44
0
 def run(self):
     a = arange(3)
     assert_array_equal(evaluate('a**3'), array([0, 1, 8]))
示例#45
0
 def test_compare_constant(self):
     sarr = self.str_array1
     expr = 'sarr >= %r' % self.str_constant
     res1 = eval(expr)
     res2 = evaluate(expr)
     assert_array_equal(res1, res2)
示例#46
0
 def test_small_long(self):
     # Small longs should not be downgraded to ints.
     res = evaluate('42L')
     assert_array_equal(res, 42)
     self.assertEqual(res.dtype.name, 'int64')
示例#47
0
 def test_rational_expr(self):
     a = arange(1e5)
     b = arange(1e5) * 0.1
     x = (a + 2 * b) / (1 + a + 4 * b * b)
     y = evaluate("(a + 2*b) / (1 + a + 4*b*b)")
     assert_array_equal(x, y)
示例#48
0
 def test_simple(self):
     a = array([1., 2., 3.])
     b = array([4., 5., 6.])
     c = array([7., 8., 9.])
     x = evaluate("2*a + 3*b*c")
     assert_array_equal(x, array([  86.,  124.,  168.]))
示例#49
0
 def test_big_int(self):
     # Big ints should be promoted to longs.
     # This test may only fail under 64-bit platforms.
     res = evaluate('2**40')
     assert_array_equal(res, 2**40)
     self.assertEqual(res.dtype.name, 'int64')
示例#50
0
 def test_reductions(self):
     # Check that they compile OK.
     assert_equal(
         disassemble(numexpr("sum(x**2+2, axis=None)", [('x', float)])),
         [('mul_fff', 't3', 'r1[x]', 'r1[x]'),
          ('add_fff', 't3', 't3', 'c2[2.0]'),
          ('sum_ffn', 'r0', 't3', None)])
     assert_equal(
         disassemble(numexpr("sum(x**2+2, axis=1)", [('x', float)])),
         [('mul_fff', 't3', 'r1[x]', 'r1[x]'),
          ('add_fff', 't3', 't3', 'c2[2.0]'), ('sum_ffn', 'r0', 't3', 1)])
     assert_equal(
         disassemble(numexpr("prod(x**2+2, axis=2)", [('x', float)])),
         [('mul_fff', 't3', 'r1[x]', 'r1[x]'),
          ('add_fff', 't3', 't3', 'c2[2.0]'), ('prod_ffn', 'r0', 't3', 2)])
     # Check that full reductions work.
     x = arange(10.0)
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0))
     # Check that reductions along an axis work
     y = arange(9.0).reshape(3, 3)
     assert_equal(evaluate("sum(y**2, axis=1)"), sum(y**2, axis=1))
     assert_equal(evaluate("sum(y**2, axis=0)"), sum(y**2, axis=0))
     assert_equal(evaluate("sum(y**2, axis=None)"), sum(y**2, axis=None))
     assert_equal(evaluate("prod(y**2, axis=1)"), prod(y**2, axis=1))
     assert_equal(evaluate("prod(y**2, axis=0)"), prod(y**2, axis=0))
     assert_equal(evaluate("prod(y**2, axis=None)"), prod(y**2, axis=None))
     # Check integers
     x = x.astype(int)
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0))
     # Check longs
     x = x.astype(long)
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0))
     # Check complex
     x = x + 5j
     assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0))
     assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0))
示例#51
0
def find_colinear_hits(blastfile, qeval, seval, mask='query', as_str=False):
    sqlite_file = blastfile[:blastfile.rfind(".")] + ".sqlite"
    db = sqlite3.connect(sqlite_file)
    cur = db.cursor()
    
    # need these to convert the absolute coords in sqlite to match
    # the local positions in the fresh blast
    qmin, smin = [x[0] for x in cur.execute('SELECT bpmin FROM image_info ORDER BY id')]

    # so we mask the new blast with anything that's NOT an HSP
    sql = "SELECT image_id, bpmin, bpmax FROM image_data WHERE type != 'HSP'"
    if mask == 'query': sql += ' AND image_id = 1'

    b = blast_array(blastfile, dopickle=False, best_hit=0, maxkeep=99999)
    if plot:
        pylab.plot(b['qstart'], b['sstart'], "kx")
    b = b[(b['eval'] < qeval) & (b['eval'] < seval)]

    if plot:
        pylab.plot(b['qstart'], b['sstart'], "ro")

    # TODO: remove stuff that's way off the diagonal?


    for row  in cur.execute(sql).fetchall():
        (start, stop, lmin) = row[0] == 1 and ('qstart', 'qstop', qmin) or ('sstart', 'sstop', smin)
        assert (start, stop) == ('qstart', 'qstop') # for now, always only using query
        cds_start, cds_stop = row[1] - lmin + 1, row[2] - lmin + 1
        bstart, bstop  = b[start], b[stop]
        b = b[numexpr.evaluate("(((bstart < cds_start) & (bstop < cds_start)) | ((bstop  > cds_stop ) & (bstart > cds_stop)))")]
                               
    r = 0 
    if not b.shape[0]: return None
    delta = 0.2 * b['sstart'].max()

    # here, try to find a sort of line, and keep removing outliers to only get linear cnss
    for i in range(4):
        slope, intercept, r, zy, zz = linregress(b['qstart'], b['sstart'])
        #print >>sys.stderr, slope, intercept, r, zy, zz
        if r > 0.8: break
        bqstart = b['qstart']
        expected = numexpr.evaluate('intercept + slope * bqstart')
        bsstart = b['sstart']
        s = b.shape[0]
        b = b[numexpr.evaluate('bsstart - expected < delta')]
        if s == b.shape[0]: break # not removing anything.

    if plot:
        pylab.plot(b['qstart'], b['sstart'], "bo")
        pylab.savefig('/var/www/ms_tmp/d.png')
    
    cnss = []
    start_stops = [map(lambda p: int(p) + qmin - 1, pair) for pair in zip(b['qstart'], b['qstop'])]

    for qstart, qstop in start_stops:
        qres =  cur.execute('SELECT xmin, ymin, xmax, ymax, id, pair_id FROM image_data WHERE image_id = 1 AND bpmin = ? AND bpmax = ?', (qstart, qstop)).fetchone()
        this_cns = [qres[:-1]]
        if not qres: continue
        sres =  cur.execute('SELECT xmin, ymin, xmax, ymax, id FROM image_data WHERE id = ?', (qres[-1],)).fetchone()
        this_cns.append(sres)
        cnss.append(this_cns)
    return cnss
示例#52
0
def find_colinear_hits(blastfile, qeval, seval, mask='query', as_str=False):
    sqlite_file = blastfile[:blastfile.rfind(".")] + ".sqlite"
    db = sqlite3.connect(sqlite_file)
    cur = db.cursor()

    # need these to convert the absolute coords in sqlite to match
    # the local positions in the fresh blast
    qmin, smin = [
        x[0] for x in cur.execute('SELECT bpmin FROM image_info ORDER BY id')
    ]

    # so we mask the new blast with anything that's NOT an HSP
    sql = "SELECT image_id, bpmin, bpmax FROM image_data WHERE type != 'HSP'"
    if mask == 'query': sql += ' AND image_id = 1'

    b = blast_array(blastfile, dopickle=False, best_hit=0, maxkeep=99999)
    if plot:
        pylab.plot(b['qstart'], b['sstart'], "kx")
    b = b[(b['eval'] < qeval) & (b['eval'] < seval)]

    if plot:
        pylab.plot(b['qstart'], b['sstart'], "ro")

    # TODO: remove stuff that's way off the diagonal?

    for row in cur.execute(sql).fetchall():
        (start, stop,
         lmin) = row[0] == 1 and ('qstart', 'qstop', qmin) or ('sstart',
                                                               'sstop', smin)
        assert (start, stop) == ('qstart', 'qstop'
                                 )  # for now, always only using query
        cds_start, cds_stop = row[1] - lmin + 1, row[2] - lmin + 1
        bstart, bstop = b[start], b[stop]
        b = b[numexpr.evaluate(
            "(((bstart < cds_start) & (bstop < cds_start)) | ((bstop  > cds_stop ) & (bstart > cds_stop)))"
        )]

    r = 0
    if not b.shape[0]: return None
    delta = 0.2 * b['sstart'].max()

    # here, try to find a sort of line, and keep removing outliers to only get linear cnss
    for i in range(4):
        slope, intercept, r, zy, zz = linregress(b['qstart'], b['sstart'])
        #print >>sys.stderr, slope, intercept, r, zy, zz
        if r > 0.8: break
        bqstart = b['qstart']
        expected = numexpr.evaluate('intercept + slope * bqstart')
        bsstart = b['sstart']
        s = b.shape[0]
        b = b[numexpr.evaluate('bsstart - expected < delta')]
        if s == b.shape[0]: break  # not removing anything.

    if plot:
        pylab.plot(b['qstart'], b['sstart'], "bo")
        pylab.savefig('/var/www/ms_tmp/d.png')

    cnss = []
    start_stops = [
        map(lambda p: int(p) + qmin - 1, pair)
        for pair in zip(b['qstart'], b['qstop'])
    ]

    for qstart, qstop in start_stops:
        qres = cur.execute(
            'SELECT xmin, ymin, xmax, ymax, id, pair_id FROM image_data WHERE image_id = 1 AND bpmin = ? AND bpmax = ?',
            (qstart, qstop)).fetchone()
        this_cns = [qres[:-1]]
        if not qres: continue
        sres = cur.execute(
            'SELECT xmin, ymin, xmax, ymax, id FROM image_data WHERE id = ?',
            (qres[-1], )).fetchone()
        this_cns.append(sres)
        cnss.append(this_cns)
    return cnss