class Basic10DTestCase(BasicTestCase):
# 10D case
title = "Rank-10 case 1"
tupleInt = ones((2, ) * 10, 'Int32')
# The next tuple consumes far more time, so this
# test should be run in common.heavy mode.
# Dimensions greather than 6 in numarray strings gives some warnings
tupleChar = strings.array("abc" * 2**6, shape=(2, ) * 6, itemsize=3)
def test01_readTableChar(self):
"""Checking column conversion into numarray in read(). Chars."""
table = self.fileh.root.table
table.flavor = "numarray"
for colname in table.colnames:
numcol = table.read(field=colname)
typecol = table.coltypes[colname]
itemsizecol = table.description._v_dtypes[colname].base.itemsize
if typecol == "string":
if itemsizecol > 1:
orignumcol = strings.array(['abcd']*self.nrows, itemsize=4)
else:
orignumcol = strings.array(['a']*self.nrows, itemsize=1)
if common.verbose:
print "Itemsize of column:", itemsizecol
print "Shape of numarray column read:", numcol.shape
print "Should look like:", orignumcol.shape
print "First 3 elements of read col:", numcol[:3]
# Check that both numarray objects are equal
self.assertTrue(allequal(numcol, orignumcol, "numarray"))
def test01_readTableChar(self):
"""Checking column conversion into numarray in read(). Chars."""
table = self.fileh.root.table
table.flavor = "numarray"
for colname in table.colnames:
numcol = table.read(field=colname)
typecol = table.coltypes[colname]
itemsizecol = table.description._v_dtypes[colname].base.itemsize
if typecol == "string":
if itemsizecol > 1:
orignumcol = strings.array(['abcd']*self.nrows, itemsize=4)
else:
orignumcol = strings.array(['a']*self.nrows, itemsize=1)
if common.verbose:
print "Itemsize of column:", itemsizecol
print "Shape of numarray column read:", numcol.shape
print "Should look like:", orignumcol.shape
print "First 3 elements of read col:", numcol[:3]
# Check that both numarray objects are equal
assert allequal(numcol, orignumcol, "numarray")
def test01_char_nc(self):
"Data integrity during recovery (non-contiguous character objects)"
a = strings.array(self.tupleChar)
if a.shape == ():
b = a # We cannot use the indexing notation
else:
b = a[::2]
# Ensure that this numarray string is non-contiguous
if a.shape[0] > 2:
self.assertEqual(b.iscontiguous(), 0)
self.WriteRead(b)
return
def test01_char_nc(self):
"Data integrity during recovery (non-contiguous character objects)"
a = strings.array(self.tupleChar)
if a.shape == ():
b = a # We cannot use the indexing notation
else:
b = a[::2]
# Ensure that this numarray string is non-contiguous
if a.shape[0] > 2:
self.assertEqual(b.iscontiguous(), 0)
self.WriteRead(b)
return
def setUp(self):
'''IDL: instantiate a pyIDL session'''
self.session = IDL(stdout=False)
self.int = 1
self.list = [1, 2]
self.array = array(self.list)
self.dict = {}
self.matrix = [[1, 2, 3], [4, 5, 6]] #XXX: add this to ALL/MOST tests!
self.none = None
self.str = 'foo'
self.bytearray = array(self.str)
self.strlist = ["hello", "world"]
self.chararray = numstring.array(self.strlist)
return #FIXME: do I want a new session for each test?
def test00_char(self):
"Data integrity during recovery (character objects)"
a = strings.array(self.tupleChar)
self.WriteRead(a)
return
bar.done = total
bar.show()
print ''
return record
#############
# Test Code
#############
if __name__ == '__main__':
r = Master('./lahman52/Master.csv')
b = Batting('./lahman52/Batting.csv')
p = Pitching('./lahman52/Pitching.csv')
print 'Good.'
sarray = NS.array(r.field('nameLast'))
tuple = sarray.search('Nomo')
idx = tuple[0][0]
playerID = r.field('playerID')[idx]
sarray = NS.array(b.field('playerID'))
tuple = sarray.search(playerID)
print tuple
for each in tuple[0]:
print b[each]
sarray = NS.array(p.field('playerID'))
tuple = sarray.search(playerID)
def createFile(filename, nrows, filters, indexmode, heavy, noise, bfile,
verbose):
# Initialize some variables
t1 = 0.; t2 = 0.
tcpu1 = 0.; tcpu2 = 0.
rowsecf = 0.; rowseci = 0.
size1 = 0.; size2 = 0.
if indexmode == "standard":
print "Creating a new database:", dbfile
instd=os.popen("/usr/local/bin/sqlite "+dbfile, "w")
CREATESTD="""
CREATE TABLE small (
-- Name Type -- Example
---------------------------------------
recnum INTEGER PRIMARY KEY, -- 345
var1 char(4), -- Abronia villosa
var2 INTEGER, -- 111
var3 FLOAT -- 12.32
);
"""
CREATEIDX="""
CREATE TABLE small (
-- Name Type -- Example
---------------------------------------
recnum INTEGER PRIMARY KEY, -- 345
var1 char(4), -- Abronia villosa
var2 INTEGER, -- 111
var3 FLOAT -- 12.32
);
CREATE INDEX ivar1 ON small(var1);
CREATE INDEX ivar2 ON small(var2);
CREATE INDEX ivar3 ON small(var3);
"""
# Creating the table first and indexing afterwards is a bit faster
instd.write(CREATESTD)
instd.close()
conn = sqlite.connect(dbfile)
cursor = conn.cursor()
if indexmode == "standard":
place_holders = ",".join(['%s']*3)
# Insert rows
SQL = "insert into small values(NULL, %s)" % place_holders
time1 = time.time()
cpu1 = time.clock()
# This way of filling is to copy the PyTables benchmark
nrowsbuf = 1000
minimum = 0
maximum = nrows
for i in xrange(0, nrows, nrowsbuf):
if i+nrowsbuf > nrows:
j = nrows
else:
j = i+nrowsbuf
if randomvalues:
var3 = random_array.uniform(minimum, maximum, shape=[j-i])
else:
var3 = numarray.arange(i, j, type=numarray.Float64)
if noise:
var3 += random_array.uniform(-3, 3, shape=[j-i])
var2 = numarray.array(var3, type=numarray.Int32)
var1 = strings.array(None, shape=[j-i], itemsize=4)
if not heavy:
for n in xrange(j-i):
var1[n] = str("%.4s" % var2[n])
for n in xrange(j-i):
fields = (var1[n], var2[n], var3[n])
cursor.execute(SQL, fields)
conn.commit()
t1 = round(time.time()-time1, 5)
tcpu1 = round(time.clock()-cpu1, 5)
rowsecf = nrows/t1
size1 = os.stat(dbfile)[6]
print "******** Results for writing nrows = %s" % (nrows), "*********"
print "Insert time:", t1, ", KRows/s:", round((nrows/10.**3)/t1, 3),
print ", File size:", round(size1/(1024.*1024.), 3), "MB"
# Indexem
if indexmode == "indexed":
time1 = time.time()
cpu1 = time.clock()
if not heavy:
cursor.execute("CREATE INDEX ivar1 ON small(var1)")
conn.commit()
cursor.execute("CREATE INDEX ivar2 ON small(var2)")
conn.commit()
cursor.execute("CREATE INDEX ivar3 ON small(var3)")
conn.commit()
t2 = round(time.time()-time1, 5)
tcpu2 = round(time.clock()-cpu1, 5)
rowseci = nrows/t2
print "Index time:", t2, ", IKRows/s:", round((nrows/10.**3)/t2, 3),
size2 = os.stat(dbfile)[6] - size1
print ", Final size with index:", round(size2/(1024.*1024), 3), "MB"
conn.close()
# Collect benchmark data
bf = openFile(bfile, "a")
recsize = "sqlite_small"
if indexmode == "indexed":
table = bf.getNode("/"+recsize+"/create_indexed")
else:
table = bf.getNode("/"+recsize+"/create_standard")
table.row["nrows"] = nrows
table.row["irows"] = nrows
table.row["tfill"] = t1
table.row["tidx"] = t2
table.row["tcfill"] = tcpu1
table.row["tcidx"] = tcpu2
table.row["psyco"] = psycon
table.row["rowsecf"] = rowsecf
table.row["rowseci"] = rowseci
table.row["fsize"] = size1
table.row["isize"] = size2
table.row.append()
bf.close()
return
def test00_char(self):
"Data integrity during recovery (character objects)"
a = strings.array(self.tupleChar)
self.WriteRead(a)
return
