def assert_missing_op(self, op, expr, local_dict): msg = "expected NotImplementedError regarding '%s'" % op try: evaluate(expr, local_dict) except NotImplementedError, nie: if "'%s'" % op not in nie.args[0]: self.fail(msg)
def test_illegal_value(self): a = arange(3) try: evaluate("a < [0, 0, 0]") except TypeError: pass else: self.fail()
def test_broadcasting(self): a = arange(100).reshape(10, 10)[::2] c = arange(10) d = arange(5).reshape(5, 1) assert_array_equal(evaluate("a+c"), a + c) assert_array_equal(evaluate("a+d"), a + d) expr = numexpr("2.0*a+3.0*c", [('a', float), ('c', float)]) assert_array_equal(expr(a, c), 2.0 * a + 3.0 * c)
def test_broadcasting(self): a = arange(100).reshape(10,10)[::2] c = arange(10) d = arange(5).reshape(5,1) assert_array_equal(evaluate("a+c"), a+c) assert_array_equal(evaluate("a+d"), a+d) expr = numexpr("2.0*a+3.0*c",[('a',float),('c', float)]) assert_array_equal(expr(a,c), 2.0*a+3.0*c)
def test_long_constant_promotion(self): int32array = arange(100, dtype='int32') res = int32array * 2 res32 = evaluate('int32array * 2') res64 = evaluate('int32array * 2L') assert_array_equal(res, res32) assert_array_equal(res, res64) self.assertEqual(res32.dtype.name, 'int32') self.assertEqual(res64.dtype.name, 'int64')
def test_complex_strides(self): a = arange(100).reshape(10,10)[::2] b = arange(50).reshape(5,10) assert_array_equal(evaluate("a+b"), a+b) c = empty([10], dtype=[('c1', int32), ('c2', uint16)]) c['c1'] = arange(10) c['c2'].fill(0xaaaa) c1 = c['c1'] a0 = a[0] assert_array_equal(evaluate("c1"), c1) assert_array_equal(evaluate("a0+c1"), a0+c1)
def test_complex_strides(self): a = arange(100).reshape(10, 10)[::2] b = arange(50).reshape(5, 10) assert_array_equal(evaluate("a+b"), a + b) c = empty([10], dtype=[('c1', int32), ('c2', uint16)]) c['c1'] = arange(10) c['c2'].fill(0xaaaa) c1 = c['c1'] a0 = a[0] assert_array_equal(evaluate("c1"), c1) assert_array_equal(evaluate("a0+c1"), a0 + c1)
def test_axis(self): y = arange(9.0).reshape(3, 3) try: evaluate("sum(y, axis=2)") except ValueError: pass else: raise ValueError("should raise exception!") try: evaluate("sum(y, axis=-3)") except ValueError: pass else: raise ValueError("should raise exception!")
def test_select(self): f0 = arange(10, dtype=int32) f1 = arange(10, dtype=float64) irregular = rec.fromarrays([f0, f1]) f0 = irregular['f0'] f1 = irregular['f1'] i0 = evaluate('f0 < 5') i1 = evaluate('f1 < 5') assert_array_equal(f0[i0], arange(5, dtype=int32)) assert_array_equal(f1[i1], arange(5, dtype=float64))
def test_axis(self): y = arange(9.0).reshape(3,3) try: evaluate("sum(y, axis=2)") except ValueError: pass else: raise ValueError("should raise exception!") try: evaluate("sum(y, axis=-3)") except ValueError: pass else: raise ValueError("should raise exception!")
def test_int64_array_promotion(self): int32array = arange(100, dtype='int32') int64array = arange(100, dtype='int64') respy = int32array * int64array resnx = evaluate('int32array * int64array') assert_array_equal(respy, resnx) self.assertEqual(resnx.dtype.name, 'int64')
def test_null_chars(self): str_list = [ '\0\0\0', '\0\0foo\0', '\0\0foo\0b', '\0\0foo\0b\0', 'foo\0', 'foo\0b', 'foo\0b\0', 'foo\0bar\0baz\0\0' ] for s in str_list: r = evaluate('s') self.assertEqual(s, r.tostring()) # check *all* stored data
def test_compare_array(self): sarr1 = self.str_array1 sarr2 = self.str_array2 expr = 'sarr1 >= sarr2' res1 = eval(expr) res2 = evaluate(expr) assert_array_equal(res1, res2)
def test_compare_variable(self): sarr = self.str_array1 svar = self.str_constant expr = 'sarr >= svar' res1 = eval(expr) res2 = evaluate(expr) assert_array_equal(res1, res2)
def test_complex_expr(self): def complex(a, b): c = zeros(a.shape, dtype=complex_) c.real = a c.imag = b return c a = arange(1e4) b = arange(1e4)**1e-5 z = a + 1j*b x = z.imag x = sin(complex(a, b)).real + z.imag y = evaluate("sin(complex(a, b)).real + z.imag") assert_array_almost_equal(x, y)
def test_complex_expr(self): def complex(a, b): c = zeros(a.shape, dtype=complex_) c.real = a c.imag = b return c a = arange(1e4) b = arange(1e4)**1e-5 z = a + 1j * b x = z.imag x = sin(complex(a, b)).real + z.imag y = evaluate("sin(complex(a, b)).real + z.imag") assert_array_almost_equal(x, y)
def method(self): npval = eval(expr, globals(), this_locals) try: neval = evaluate(expr, local_dict=this_locals, optimization=optimization) assert equal(npval, neval, exact), \ """%r (test_scalar=%r, dtype=%r, optimization=%r, exact=%r, npval=%r (%r), neval=%r (%r))""" % (expr, test_scalar, dtype.__name__, optimization, exact, npval, type(npval), neval, type(neval)) except AssertionError: raise except NotImplementedError: print('%r not implemented for %s' % (expr,dtype.__name__)) except: print('numexpr error for expression %r' % (expr,)) raise
def method(self): npval = eval(expr, globals(), this_locals) try: neval = evaluate(expr, local_dict=this_locals, optimization=optimization) assert equal(npval, neval, exact), \ """%r (test_scalar=%r, dtype=%r, optimization=%r, exact=%r, npval=%r (%r), neval=%r (%r))""" % (expr, test_scalar, dtype.__name__, optimization, exact, npval, type(npval), neval, type(neval)) except AssertionError: raise except NotImplementedError: print('%r not implemented for %s' % (expr, dtype.__name__)) except: print('numexpr error for expression %r' % (expr, )) raise
def test_compare_prefix(self): # Check comparing two strings where one is a prefix of the # other. for s1, s2 in [ ('foo', 'foobar'), ('foo', 'foo\0bar'), ('foo\0a', 'foo\0bar') ]: self.assert_(evaluate('s1 < s2')) self.assert_(evaluate('s1 <= s2')) self.assert_(evaluate('~(s1 == s2)')) self.assert_(evaluate('~(s1 >= s2)')) self.assert_(evaluate('~(s1 > s2)')) # Check for NumPy array-style semantics in string equality. s1, s2 = 'foo', 'foo\0\0' self.assert_(evaluate('s1 == s2'))
def test_compare_prefix(self): # Check comparing two strings where one is a prefix of the # other. for s1, s2 in [('foo', 'foobar'), ('foo', 'foo\0bar'), ('foo\0a', 'foo\0bar')]: self.assert_(evaluate('s1 < s2')) self.assert_(evaluate('s1 <= s2')) self.assert_(evaluate('~(s1 == s2)')) self.assert_(evaluate('~(s1 >= s2)')) self.assert_(evaluate('~(s1 > s2)')) # Check for NumPy array-style semantics in string equality. s1, s2 = 'foo', 'foo\0\0' self.assert_(evaluate('s1 == s2'))
def run(self): a = arange(3) assert_array_equal(evaluate('a**3'), array([0, 1, 8]))
def test_big_int(self): # Big ints should be promoted to longs. # This test may only fail under 64-bit platforms. res = evaluate('2**40') assert_array_equal(res, 2**40) self.assertEqual(res.dtype.name, 'int64')
def test_simple_expr(self): x = arange(1e5) y = evaluate("x") assert_array_equal(x, y)
def test_simple_expr_small_array(self): x = arange(100.0) y = evaluate("x") assert_array_equal(x, y)
def test_all_scalar(self): a = 3. b = 4. assert_equal(evaluate("a+b"), a+b) expr = numexpr("2*a+3*b",[('a',float),('b', float)]) assert_equal(expr(a,b), 2*a+3*b)
def test_compare_copy(self): sarr = self.str_array1 expr = 'sarr' res1 = eval(expr) res2 = evaluate(expr) assert_array_equal(res1, res2)
def test_all_scalar(self): a = 3. b = 4. assert_equal(evaluate("a+b"), a + b) expr = numexpr("2*a+3*b", [('a', float), ('b', float)]) assert_equal(expr(a, b), 2 * a + 3 * b)
def test_simple(self): a = array([1., 2., 3.]) b = array([4., 5., 6.]) c = array([7., 8., 9.]) x = evaluate("2*a + 3*b*c") assert_array_equal(x, array([86., 124., 168.]))
def test_small_long(self): # Small longs should not be downgraded to ints. res = evaluate('42L') assert_array_equal(res, 42) self.assertEqual(res.dtype.name, 'int64')
def test_rational_expr(self): a = arange(1e5) b = arange(1e5) * 0.1 x = (a + 2*b) / (1 + a + 4*b*b) y = evaluate("(a + 2*b) / (1 + a + 4*b*b)") assert_array_equal(x, y)
def test_reductions(self): # Check that they compile OK. assert_equal(disassemble( numexpr("sum(x**2+2, axis=None)", [('x', float)])), [('mul_fff', 't3', 'r1[x]', 'r1[x]'), ('add_fff', 't3', 't3', 'c2[2.0]'), ('sum_ffn', 'r0', 't3', None)]) assert_equal(disassemble( numexpr("sum(x**2+2, axis=1)", [('x', float)])), [('mul_fff', 't3', 'r1[x]', 'r1[x]'), ('add_fff', 't3', 't3', 'c2[2.0]'), ('sum_ffn', 'r0', 't3', 1)]) assert_equal(disassemble( numexpr("prod(x**2+2, axis=2)", [('x', float)])), [('mul_fff', 't3', 'r1[x]', 'r1[x]'), ('add_fff', 't3', 't3', 'c2[2.0]'), ('prod_ffn', 'r0', 't3', 2)]) # Check that full reductions work. x = arange(10.0) assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0)) # Check that reductions along an axis work y = arange(9.0).reshape(3,3) assert_equal(evaluate("sum(y**2, axis=1)"), sum(y**2, axis=1)) assert_equal(evaluate("sum(y**2, axis=0)"), sum(y**2, axis=0)) assert_equal(evaluate("sum(y**2, axis=None)"), sum(y**2, axis=None)) assert_equal(evaluate("prod(y**2, axis=1)"), prod(y**2, axis=1)) assert_equal(evaluate("prod(y**2, axis=0)"), prod(y**2, axis=0)) assert_equal(evaluate("prod(y**2, axis=None)"), prod(y**2, axis=None)) # Check integers x = x.astype(int) assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0)) # Check longs x = x.astype(long) assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0)) # Check complex x = x + 5j assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2+2,axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2+2,axis=0))
def test_compare_constant(self): sarr = self.str_array1 expr = 'sarr >= %r' % self.str_constant res1 = eval(expr) res2 = evaluate(expr) assert_array_equal(res1, res2)
def test_rational_expr(self): a = arange(1e5) b = arange(1e5) * 0.1 x = (a + 2 * b) / (1 + a + 4 * b * b) y = evaluate("(a + 2*b) / (1 + a + 4*b*b)") assert_array_equal(x, y)
def test_simple(self): a = array([1., 2., 3.]) b = array([4., 5., 6.]) c = array([7., 8., 9.]) x = evaluate("2*a + 3*b*c") assert_array_equal(x, array([ 86., 124., 168.]))
def test_reductions(self): # Check that they compile OK. assert_equal( disassemble(numexpr("sum(x**2+2, axis=None)", [('x', float)])), [('mul_fff', 't3', 'r1[x]', 'r1[x]'), ('add_fff', 't3', 't3', 'c2[2.0]'), ('sum_ffn', 'r0', 't3', None)]) assert_equal( disassemble(numexpr("sum(x**2+2, axis=1)", [('x', float)])), [('mul_fff', 't3', 'r1[x]', 'r1[x]'), ('add_fff', 't3', 't3', 'c2[2.0]'), ('sum_ffn', 'r0', 't3', 1)]) assert_equal( disassemble(numexpr("prod(x**2+2, axis=2)", [('x', float)])), [('mul_fff', 't3', 'r1[x]', 'r1[x]'), ('add_fff', 't3', 't3', 'c2[2.0]'), ('prod_ffn', 'r0', 't3', 2)]) # Check that full reductions work. x = arange(10.0) assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0)) # Check that reductions along an axis work y = arange(9.0).reshape(3, 3) assert_equal(evaluate("sum(y**2, axis=1)"), sum(y**2, axis=1)) assert_equal(evaluate("sum(y**2, axis=0)"), sum(y**2, axis=0)) assert_equal(evaluate("sum(y**2, axis=None)"), sum(y**2, axis=None)) assert_equal(evaluate("prod(y**2, axis=1)"), prod(y**2, axis=1)) assert_equal(evaluate("prod(y**2, axis=0)"), prod(y**2, axis=0)) assert_equal(evaluate("prod(y**2, axis=None)"), prod(y**2, axis=None)) # Check integers x = x.astype(int) assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0)) # Check longs x = x.astype(long) assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0)) # Check complex x = x + 5j assert_equal(evaluate("sum(x**2+2,axis=0)"), sum(x**2 + 2, axis=0)) assert_equal(evaluate("prod(x**2+2,axis=0)"), prod(x**2 + 2, axis=0))
def find_colinear_hits(blastfile, qeval, seval, mask='query', as_str=False): sqlite_file = blastfile[:blastfile.rfind(".")] + ".sqlite" db = sqlite3.connect(sqlite_file) cur = db.cursor() # need these to convert the absolute coords in sqlite to match # the local positions in the fresh blast qmin, smin = [x[0] for x in cur.execute('SELECT bpmin FROM image_info ORDER BY id')] # so we mask the new blast with anything that's NOT an HSP sql = "SELECT image_id, bpmin, bpmax FROM image_data WHERE type != 'HSP'" if mask == 'query': sql += ' AND image_id = 1' b = blast_array(blastfile, dopickle=False, best_hit=0, maxkeep=99999) if plot: pylab.plot(b['qstart'], b['sstart'], "kx") b = b[(b['eval'] < qeval) & (b['eval'] < seval)] if plot: pylab.plot(b['qstart'], b['sstart'], "ro") # TODO: remove stuff that's way off the diagonal? for row in cur.execute(sql).fetchall(): (start, stop, lmin) = row[0] == 1 and ('qstart', 'qstop', qmin) or ('sstart', 'sstop', smin) assert (start, stop) == ('qstart', 'qstop') # for now, always only using query cds_start, cds_stop = row[1] - lmin + 1, row[2] - lmin + 1 bstart, bstop = b[start], b[stop] b = b[numexpr.evaluate("(((bstart < cds_start) & (bstop < cds_start)) | ((bstop > cds_stop ) & (bstart > cds_stop)))")] r = 0 if not b.shape[0]: return None delta = 0.2 * b['sstart'].max() # here, try to find a sort of line, and keep removing outliers to only get linear cnss for i in range(4): slope, intercept, r, zy, zz = linregress(b['qstart'], b['sstart']) #print >>sys.stderr, slope, intercept, r, zy, zz if r > 0.8: break bqstart = b['qstart'] expected = numexpr.evaluate('intercept + slope * bqstart') bsstart = b['sstart'] s = b.shape[0] b = b[numexpr.evaluate('bsstart - expected < delta')] if s == b.shape[0]: break # not removing anything. if plot: pylab.plot(b['qstart'], b['sstart'], "bo") pylab.savefig('/var/www/ms_tmp/d.png') cnss = [] start_stops = [map(lambda p: int(p) + qmin - 1, pair) for pair in zip(b['qstart'], b['qstop'])] for qstart, qstop in start_stops: qres = cur.execute('SELECT xmin, ymin, xmax, ymax, id, pair_id FROM image_data WHERE image_id = 1 AND bpmin = ? AND bpmax = ?', (qstart, qstop)).fetchone() this_cns = [qres[:-1]] if not qres: continue sres = cur.execute('SELECT xmin, ymin, xmax, ymax, id FROM image_data WHERE id = ?', (qres[-1],)).fetchone() this_cns.append(sres) cnss.append(this_cns) return cnss
def find_colinear_hits(blastfile, qeval, seval, mask='query', as_str=False): sqlite_file = blastfile[:blastfile.rfind(".")] + ".sqlite" db = sqlite3.connect(sqlite_file) cur = db.cursor() # need these to convert the absolute coords in sqlite to match # the local positions in the fresh blast qmin, smin = [ x[0] for x in cur.execute('SELECT bpmin FROM image_info ORDER BY id') ] # so we mask the new blast with anything that's NOT an HSP sql = "SELECT image_id, bpmin, bpmax FROM image_data WHERE type != 'HSP'" if mask == 'query': sql += ' AND image_id = 1' b = blast_array(blastfile, dopickle=False, best_hit=0, maxkeep=99999) if plot: pylab.plot(b['qstart'], b['sstart'], "kx") b = b[(b['eval'] < qeval) & (b['eval'] < seval)] if plot: pylab.plot(b['qstart'], b['sstart'], "ro") # TODO: remove stuff that's way off the diagonal? for row in cur.execute(sql).fetchall(): (start, stop, lmin) = row[0] == 1 and ('qstart', 'qstop', qmin) or ('sstart', 'sstop', smin) assert (start, stop) == ('qstart', 'qstop' ) # for now, always only using query cds_start, cds_stop = row[1] - lmin + 1, row[2] - lmin + 1 bstart, bstop = b[start], b[stop] b = b[numexpr.evaluate( "(((bstart < cds_start) & (bstop < cds_start)) | ((bstop > cds_stop ) & (bstart > cds_stop)))" )] r = 0 if not b.shape[0]: return None delta = 0.2 * b['sstart'].max() # here, try to find a sort of line, and keep removing outliers to only get linear cnss for i in range(4): slope, intercept, r, zy, zz = linregress(b['qstart'], b['sstart']) #print >>sys.stderr, slope, intercept, r, zy, zz if r > 0.8: break bqstart = b['qstart'] expected = numexpr.evaluate('intercept + slope * bqstart') bsstart = b['sstart'] s = b.shape[0] b = b[numexpr.evaluate('bsstart - expected < delta')] if s == b.shape[0]: break # not removing anything. if plot: pylab.plot(b['qstart'], b['sstart'], "bo") pylab.savefig('/var/www/ms_tmp/d.png') cnss = [] start_stops = [ map(lambda p: int(p) + qmin - 1, pair) for pair in zip(b['qstart'], b['qstop']) ] for qstart, qstop in start_stops: qres = cur.execute( 'SELECT xmin, ymin, xmax, ymax, id, pair_id FROM image_data WHERE image_id = 1 AND bpmin = ? AND bpmax = ?', (qstart, qstop)).fetchone() this_cns = [qres[:-1]] if not qres: continue sres = cur.execute( 'SELECT xmin, ymin, xmax, ymax, id FROM image_data WHERE id = ?', (qres[-1], )).fetchone() this_cns.append(sres) cnss.append(this_cns) return cnss