def fill_arrays(start, stop):
col_i = numarray.arange(start, stop, type=numarray.Int32)
if userandom:
col_j = random_array.uniform(0, nrows, shape=[stop-start])
else:
col_j = numarray.array(col_i, type=numarray.Float64)
return col_i, col_j
def fill_arrays(start, stop):
col_i = numarray.arange(start, stop, type=numarray.Int32)
if userandom:
col_j = random_array.uniform(0, nrows, shape=[stop - start])
else:
col_j = numarray.array(col_i, type=numarray.Float64)
return col_i, col_j
def run(self,n_sources = 3, n_observations = 21, ra_range = [-20.0,20.0],dec_range=[-20.0,20.0],typical_err=0.3,reuse=True):
global real_list
clf()
# make the real sources
if reuse:
try:
type(real_list) == type([])
except NameError:
reuse = False
real_list = []
for ns in range(n_sources):
if not reuse:
real_list.append(source(start_pos=[random_array.uniform(ra_range[0],ra_range[1]),random_array.uniform(dec_range[0],dec_range[1])],
stype='real',start_err=[0.0,0.0],associated_obs=[]))
# print real_list[-1]
real_list[ns].plot()
constructed_source_list = []
observation_list = []
## pick a vector of len n_obsevations sources to choose from from 0 --> n_source - 1
s_start_ind = random_array.randint(0,n_sources,shape=[n_observations])
for i in range(n_observations):
# choose a real source to draw an observation from
o = obs(initial_pos=real_list[s_start_ind[i]].start_pos,true_err=[[typical_err**2,0.0],[0.0,typical_err**2]],assumed_err=[1.1*typical_err,1.1*typical_err])
o.observe_pos()
o.plot()
tmp = o.is_associated_with_source(constructed_source_list)
# print tmp
if tmp['answer'] == True:
# o.associate_with_source(tmp['sources'])
for s in tmp['best_source']:
# print (len(tmp['best_source']),o.pos,s.current_pos)
s.add_associated_obs(copy.deepcopy(o))
s.recalculate_position()
else:
## make a new source
s = source(start_pos=copy.copy(o.pos),stype='constructed',start_err=copy.copy(o.assumed_err),current_pos=copy.copy(o.pos),\
current_err=copy.copy(o.assumed_err),associated_obs=[copy.deepcopy(o)])
print "new source"
#print s
#print s.associated_obs
constructed_source_list.append(copy.deepcopy(s))
observation_list.append(copy.deepcopy(o))
for s in constructed_source_list:
# print s
s.plot('g^')
## do the comparisons between real and constructed sources
for ns in range(n_sources):
#real_list[ns].plot('ys')
pass
self.real_list = real_list
self.constructed_source_list = constructed_source_list
self.real_pos = (numarray.fromlist([x.start_pos for x in self.real_list]))
self.constructed_pos = (numarray.fromlist([x.current_pos for x in self.constructed_source_list]))
def createFile(filename, nrows, filters, indexmode, heavy, noise, bfile,
verbose):
# Initialize some variables
t1 = 0.; t2 = 0.
tcpu1 = 0.; tcpu2 = 0.
rowsecf = 0.; rowseci = 0.
size1 = 0.; size2 = 0.
if indexmode == "standard":
print "Creating a new database:", dbfile
instd=os.popen("/usr/local/bin/sqlite "+dbfile, "w")
CREATESTD="""
CREATE TABLE small (
-- Name Type -- Example
---------------------------------------
recnum INTEGER PRIMARY KEY, -- 345
var1 char(4), -- Abronia villosa
var2 INTEGER, -- 111
var3 FLOAT -- 12.32
);
"""
CREATEIDX="""
CREATE TABLE small (
-- Name Type -- Example
---------------------------------------
recnum INTEGER PRIMARY KEY, -- 345
var1 char(4), -- Abronia villosa
var2 INTEGER, -- 111
var3 FLOAT -- 12.32
);
CREATE INDEX ivar1 ON small(var1);
CREATE INDEX ivar2 ON small(var2);
CREATE INDEX ivar3 ON small(var3);
"""
# Creating the table first and indexing afterwards is a bit faster
instd.write(CREATESTD)
instd.close()
conn = sqlite.connect(dbfile)
cursor = conn.cursor()
if indexmode == "standard":
place_holders = ",".join(['%s']*3)
# Insert rows
SQL = "insert into small values(NULL, %s)" % place_holders
time1 = time.time()
cpu1 = time.clock()
# This way of filling is to copy the PyTables benchmark
nrowsbuf = 1000
minimum = 0
maximum = nrows
for i in xrange(0, nrows, nrowsbuf):
if i+nrowsbuf > nrows:
j = nrows
else:
j = i+nrowsbuf
if randomvalues:
var3 = random_array.uniform(minimum, maximum, shape=[j-i])
else:
var3 = numarray.arange(i, j, type=numarray.Float64)
if noise:
var3 += random_array.uniform(-3, 3, shape=[j-i])
var2 = numarray.array(var3, type=numarray.Int32)
var1 = strings.array(None, shape=[j-i], itemsize=4)
if not heavy:
for n in xrange(j-i):
var1[n] = str("%.4s" % var2[n])
for n in xrange(j-i):
fields = (var1[n], var2[n], var3[n])
cursor.execute(SQL, fields)
conn.commit()
t1 = round(time.time()-time1, 5)
tcpu1 = round(time.clock()-cpu1, 5)
rowsecf = nrows/t1
size1 = os.stat(dbfile)[6]
print "******** Results for writing nrows = %s" % (nrows), "*********"
print "Insert time:", t1, ", KRows/s:", round((nrows/10.**3)/t1, 3),
print ", File size:", round(size1/(1024.*1024.), 3), "MB"
# Indexem
if indexmode == "indexed":
time1 = time.time()
cpu1 = time.clock()
if not heavy:
cursor.execute("CREATE INDEX ivar1 ON small(var1)")
conn.commit()
cursor.execute("CREATE INDEX ivar2 ON small(var2)")
conn.commit()
cursor.execute("CREATE INDEX ivar3 ON small(var3)")
conn.commit()
t2 = round(time.time()-time1, 5)
tcpu2 = round(time.clock()-cpu1, 5)
rowseci = nrows/t2
print "Index time:", t2, ", IKRows/s:", round((nrows/10.**3)/t2, 3),
size2 = os.stat(dbfile)[6] - size1
print ", Final size with index:", round(size2/(1024.*1024), 3), "MB"
conn.close()
# Collect benchmark data
bf = openFile(bfile, "a")
recsize = "sqlite_small"
if indexmode == "indexed":
table = bf.getNode("/"+recsize+"/create_indexed")
else:
table = bf.getNode("/"+recsize+"/create_standard")
table.row["nrows"] = nrows
table.row["irows"] = nrows
table.row["tfill"] = t1
table.row["tidx"] = t2
table.row["tcfill"] = tcpu1
table.row["tcidx"] = tcpu2
table.row["psyco"] = psycon
table.row["rowsecf"] = rowsecf
table.row["rowseci"] = rowseci
table.row["fsize"] = size1
table.row["isize"] = size2
table.row.append()
bf.close()
return
def run(self,
n_sources=3,
n_observations=21,
ra_range=[-20.0, 20.0],
dec_range=[-20.0, 20.0],
typical_err=0.3,
reuse=True):
global real_list
clf()
# make the real sources
if reuse:
try:
type(real_list) == type([])
except NameError:
reuse = False
real_list = []
for ns in range(n_sources):
if not reuse:
real_list.append(
source(start_pos=[
random_array.uniform(ra_range[0], ra_range[1]),
random_array.uniform(dec_range[0], dec_range[1])
],
stype='real',
start_err=[0.0, 0.0],
associated_obs=[]))
# print real_list[-1]
real_list[ns].plot()
constructed_source_list = []
observation_list = []
## pick a vector of len n_obsevations sources to choose from from 0 --> n_source - 1
s_start_ind = random_array.randint(0,
n_sources,
shape=[n_observations])
for i in range(n_observations):
# choose a real source to draw an observation from
o = obs(initial_pos=real_list[s_start_ind[i]].start_pos,
true_err=[[typical_err**2, 0.0], [0.0, typical_err**2]],
assumed_err=[1.1 * typical_err, 1.1 * typical_err])
o.observe_pos()
o.plot()
tmp = o.is_associated_with_source(constructed_source_list)
# print tmp
if tmp['answer'] == True:
# o.associate_with_source(tmp['sources'])
for s in tmp['best_source']:
# print (len(tmp['best_source']),o.pos,s.current_pos)
s.add_associated_obs(copy.deepcopy(o))
s.recalculate_position()
else:
## make a new source
s = source(start_pos=copy.copy(o.pos),stype='constructed',start_err=copy.copy(o.assumed_err),current_pos=copy.copy(o.pos),\
current_err=copy.copy(o.assumed_err),associated_obs=[copy.deepcopy(o)])
print "new source"
#print s
#print s.associated_obs
constructed_source_list.append(copy.deepcopy(s))
observation_list.append(copy.deepcopy(o))
for s in constructed_source_list:
# print s
s.plot('g^')
## do the comparisons between real and constructed sources
for ns in range(n_sources):
#real_list[ns].plot('ys')
pass
self.real_list = real_list
self.constructed_source_list = constructed_source_list
self.real_pos = (numarray.fromlist(
[x.start_pos for x in self.real_list]))
self.constructed_pos = (numarray.fromlist(
[x.current_pos for x in self.constructed_source_list]))