def create_db(filename, nrows): class Record(tables.IsDescription): col1 = tables.Int32Col() col2 = tables.Int32Col() col3 = tables.Float64Col() col4 = tables.Float64Col() con = open_db(filename, remove=1) table = con.createTable(con.root, 'table', Record, filters=filters, expectedrows=nrows) table.indexFilters = filters step = 1000*100 scale = 0.1 t1=time() j = 0 for i in xrange(0, nrows, step): stop = (j+1)*step if stop > nrows: stop = nrows arr_f8 = numarray.arange(i, stop, type=numarray.Float64) arr_i4 = numarray.arange(i, stop, type=numarray.Int32) if userandom: arr_f8 += random_array.normal(0, stop*scale, shape=[stop-i]) arr_i4 = numarray.array(arr_f8, type=numarray.Int32) recarr = records.fromarrays([arr_i4, arr_i4, arr_f8, arr_f8]) table.append(recarr) j += 1 table.flush() ctime = time()-t1 if verbose: print "insert time:", round(ctime, 5) print "Krows/s:", round((nrows/1000.)/ctime, 5) index_db(table) close_db(con)
def create_db(filename, nrows): class Record(tables.IsDescription): col1 = tables.Int32Col() col2 = tables.Int32Col() col3 = tables.Float64Col() col4 = tables.Float64Col() con = open_db(filename, remove=1) table = con.create_table(con.root, 'table', Record, filters=filters, expectedrows=nrows) table.indexFilters = filters step = 1000*100 scale = 0.1 t1=time() j = 0 for i in xrange(0, nrows, step): stop = (j+1)*step if stop > nrows: stop = nrows arr_f8 = numarray.arange(i, stop, type=numarray.Float64) arr_i4 = numarray.arange(i, stop, type=numarray.Int32) if userandom: arr_f8 += random_array.normal(0, stop*scale, shape=[stop-i]) arr_i4 = numarray.array(arr_f8, type=numarray.Int32) recarr = records.fromarrays([arr_i4, arr_i4, arr_f8, arr_f8]) table.append(recarr) j += 1 table.flush() ctime = time()-t1 if verbose: print "insert time:", round(ctime, 5) print "Krows/s:", round((nrows/1000.)/ctime, 5) index_db(table) close_db(con)
def main(argv=None): if argv is None: argv = sys.argv if len(argv)!=6: print "Usage: %s <method> <frequency> <vmin> <vmax> <datafile>"%argv[0] return 1 hybrid = "%s_hybrid"%argv[1] simplex = "%s_simplex"%argv[1] dfile = argv[5] hybrid_out = "%s.hybrid_output"%dfile simplex_out = "%s.simplex_output"%dfile temp_in = "%s.temp_input"%dfile frequency = argv[2] vmin = argv[3] vmax = argv[4] os.system("%s %s %s %s %s 200 300 3 %s"%(hybrid,dfile,frequency,vmin,vmax,hybrid_out)) fin = open(hybrid_out,"r") params = [] paramnames = [] v = [] tin = [] tfit = [] for l in fin: tokens = l.split() if tokens[0]=='#': if tokens[1]!="Attained": paramnames.append(tokens[1]) params.append(float(tokens[2])) else: v.append(float(tokens[0])) tin.append(float(tokens[1])) tfit.append(float(tokens[2])) fin.close() tres = [tfit[i]-tin[i] for i in xrange(len(tin))] rms = reduce(lambda x,y: x+y*y, tres,0.0) rms = math.sqrt(rms/len(tres)) prmstr = ' '.join([str(i) for i in params]) rnd.seed() meanparams = [0.0]*len(params) rmsparams = [0.0]*len(params) niter = 100 for i in xrange(niter): ret = 1 while(ret!=0): rmsarr = rnd.normal(0.0,rms,len(tfit)) trand = [tfit[j]+rmsarr[j] for j in xrange(len(tfit))] fout = open(temp_in,"w") for j in range(len(v)): print >>fout,"%f\t%g"%(v[j],trand[j]) fout.close() ret=os.system("%s %s %s %s %s %s %s"%(simplex,temp_in,frequency,vmin,vmax,prmstr,simplex_out)) simplex_params = [] fin = open(simplex_out,"r") for l in fin: tokens = l.split() if tokens[0]=='#': if tokens[1]!="Attained": simplex_params.append(float(tokens[2])) fin.close() for j in xrange(len(params)): meanparams[j]+=simplex_params[j] rmsparams[j]+=simplex_params[j]*simplex_params[j] rmsparams=map(lambda x,y: math.sqrt((x-(y**2)/niter)/niter),rmsparams,meanparams) meanparams=[x/niter for x in meanparams] print "Residual RMS: ",rms print "Parameter\tMean\t\tSigma\t\tBest" for i in range(len(paramnames)): print "%s\t\t%f\t%f\t%f"%(paramnames[i],meanparams[i],rmsparams[i],params[i]) os.remove(simplex_out) os.remove(temp_in) return 0
def gen_cusum(data,reset_value=0.0): # no obvious way to vectorize this out = 0.0 for value in data: out = max(reset_value,out+value) yield out ######################################################################## if __name__ == "__main__": import doctest doctest.testmod() # batch mode timing tests import numarray.random_array as narand import time nullmodel = GaussianDistribution(mean=550.2,variance=48.2) altmodel = GaussianDistribution(mean=460.4,variance=39.2) cdmodel = ChangeDetectionModel(nullmodel,altmodel,threshold=21.0) for n in range(8): data = narand.normal(470.0,88.0,shape=10**n) _t0 = time.time() out = cdmodel(data) _t1 = time.time() secs = _t1 - _t0 print '%s seconds to process %s events (%s events/second) result=%s' \ % (secs,len(data),len(data)/secs,out) ########################################################################