def set_variable(self,lst,mtime):
'''set additional variables besides predefined environment variables
in sixdeskenv and sysenv'''
conn = self.conn
env_var = self.orig_env_var
cols=SQLTable.cols_from_fields(tables.Env.fields)
tab = SQLTable(conn,'env',cols,tables.Env.key)
flag = 0
upflag = 0
for i in lst:
if not ('env_timestamp' in str(i[0]) or str(i[0] in tables.def_var)):
if str(i[0]) in env_var.keys():
if str(i[1]) != env_var[str(i[0])]:
if is_number(str(i[1])) and is_number(env_var[str(i[0])]):
if float(str(i[1])) != float(env_var[str(i[0])]):
upflag = 1
else:
upflag = 1
if upflag == 1:
print 'variable',str(i[0]),'already present updating value from',
print env_var[str(i[0])],'to',str(i[1])
flag = 1
else:
print 'variable',str(i[0]),'not present adding'
flag = 1
env_var[str(i[0])] = str(i[1])
upflag = 0
if flag == 1:
env_var['env_timestamp']=str(time.time())
env_var = [[i,env_var[i],mtime] for i in env_var.keys()]
tab.insertl(env_var)
def st_six_results(self,env_var):
'''store fort.10 values'''
conn = self.conn
env_var = self.orig_env_var
cols = SQLTable.cols_from_fields(tables.Six_In.fields)
aff_count = 0
tab = SQLTable(conn,'six_input',cols,tables.Six_In.key)
workdir = os.path.join(env_var['sixdesktrack'],env_var['LHCDescrip'])
print "Looking for fort.10 files in %s"%workdir
rows = []
inp = tab.select("""distinct id,seed,simul,tunex,tuney,amp1,amp2,turns,
angle""")
inp = [[str(i) for i in j] for j in inp]
cols = SQLTable.cols_from_fields(tables.Six_Res.fields)
tab = SQLTable(conn,'six_results',cols,tables.Six_Res.key)
maxtime = tab.select("max(mtime)")[0][0]
if not maxtime:
maxtime = 0
cmd = "find %s -name 'fort.10.gz'"%(workdir)
a = [i for i in os.popen(cmd).read().split('\n')[:-1] if not '-' in i]
print 'fort.10 files =',len(a)
for dirName in a:
files = dirName.split('/')[-1]
dirName = dirName.replace('/fort.10.gz','')
if 'fort.10' in files and (not '-' in dirName) \
and (os.path.getmtime(dirName) > maxtime):
mtime = os.path.getmtime(dirName)
dirn = dirName.replace(workdir+'/','')
dirn = re.split('/|_',dirn)
for i in [2,3,4,5,7]:
if not ('.' in str(dirn[i])):
dirn[i] += '.0'
for i in xrange(len(inp)+1):
if i == len(inp):
print 'fort.3 file missing for',
print dirName.replace(env_var['sixdesktrack']+'/','')
print 'create file and run again'
print dirn
exit(0)
if dirn == inp[i][1:]:
six_id = inp[i][0]
break
FileObj = gzip.open(
os.path.join(dirName,files),"r").read().split("\n")[:-1]
count = 1
for lines in FileObj:
rows.append([six_id,count]+lines.split()+[mtime])
count += 1
aff_count += 1
if len(rows) > 180000:
tab.insertl(rows)
rows = []
if rows:
tab.insertl(rows)
print "no of fort.10 updated =",aff_count/30
def st_mad6t_run(self):
''' store mad run files'''
conn = self.conn
env_var = self.orig_env_var
cols = SQLTable.cols_from_fields(tables.Mad_Run.fields)
tab = SQLTable(conn,'mad6t_run',cols,tables.Mad_Run.key)
cols = SQLTable.cols_from_fields(tables.Files.fields)
tab1 = SQLTable(conn,'files',cols,tables.Files.key)
rows = {}
extra_files = []
a = []
workdir = env_var['sixtrack_input']
a = tab.select('distinct run_id')
if a:
a = [str(i[0]) for i in a]
for dirName, _, fileList in os.walk(workdir):
if 'mad.dorun' in dirName and not (dirName.split('/')[-1] in a):
print 'found new mad run',dirName.split('/')[-1]
for files in fileList:
if not (files.endswith('.mask') or 'out' in files
or files.endswith('log') or files.endswith('lsf')):
seed = files.split('.')[-1]
run_id = dirName.split('/')[-1]
mad_in = sqlite3.Binary(
compressBuf(os.path.join(dirName, files))
)
out_file = files.replace('.', '.out.')
mad_out = sqlite3.Binary(
compressBuf(os.path.join(dirName, out_file))
)
lsf_file = 'mad6t_' + seed + '.lsf'
mad_lsf = sqlite3.Binary(
compressBuf(os.path.join(dirName, lsf_file))
)
log_file = files.replace('.','_mad6t_')+'.log'
mad_log = sqlite3.Binary(
compressBuf(os.path.join(dirName, log_file))
)
time = os.path.getmtime(
os.path.join(dirName, log_file)
)
rows[seed] = []
rows[seed].append(
[run_id, seed, mad_in, mad_out, mad_lsf,
mad_log, time]
)
if files.endswith('.mask'):
path = os.path.join(dirName, files)
content = sqlite3.Binary(compressBuf(path))
path = path.replace(
env_var['scratchdir']+'/','')
extra_files.append([path, content])
if rows:
lst = dict_to_list(rows)
tab.insertl(lst)
rows = {}
if extra_files:
tab1.insertl(extra_files)
def st_boinc(self,conn):
'''store fort.10 files from boinc directory to local DB '''
env_var = self.env_var
study = env_var['LHCDescrip']
cols=SQLTable.cols_from_fields(tables.Six_Res.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'six_results',cols)
maxtime = tab.select("max(mtime)")[0][0]
if not maxtime:
maxtime = 0
cur = conn.cursor()
cur.execute("set global max_allowed_packet=209715200;")
cur.execute("set global wait_timeout=120;")
cur.execute("set global net_write_timeout=120;")
cur.execute("set global net_read_timeout=120;")
sql = "insert into six_results values (%s)"
sql = sql%(','.join("%s " for _ in xrange(len(cols))))
rows = []
boincdir = os.path.join(env_var['sixdeskboincdir'],'results')
cmd = "find %s -name '*boinc*'"%(boincdir)
a = os.popen(cmd).read().split("\n")[:-1]
for dirName in a and (os.path.getmtime(dirName) > maxtime):
mtime = os.path.getmtime(dirName)
dirn = dirName.replace(boincdir,'')
dirn = dirn.replace(env_var['sixdeskboincdirname']+"__","")
inp = re.split('_*',dirn)[:-3]
if inp[1] == 's':
inp[1] = 'simul'
inp[-2] = 'e' + inp[-2]
sql = """SELECT id from six_input where seed=? and simul=? and tunex=?
and tuney=? and amp1=? and amp2=? and turns=? and angle=?"""
six_id = []
cur.execute(sql,inp)
six_id = list(cur)
if six_id:
six_id = six_id[0][0]
else:
print 'fort.3 missing for','/'.join(inp)
exit(0)
count = 1
FileObj = open(dirName).read().split("\n")[:-1]
for lines in FileObj:
rows.append([study,six_id,count]+lines.split()+[mtime])
count += 1
if len(rows) > 150000:
cur.executemany(sql,rows)
conn.commit()
rows = []
if rows:
cur.executemany(sql,rows)
conn.commit()
def st_six_beta(self,env_var):
''' store general_input, sixdesktunes, betavalues '''
conn = self.conn
env_var = self.orig_env_var
cols = SQLTable.cols_from_fields(tables.Six_Be.fields)
tab = SQLTable(conn,'six_beta',cols,tables.Six_Be.key)
cols = SQLTable.cols_from_fields(tables.Files.fields)
tab1 = SQLTable(conn,'files',cols,tables.Files.key)
workdir = os.path.join(env_var['sixdesktrack'],env_var['LHCDescrip'])
rows = {}
extra_files = []
beta = six = gen = []
cmd = "find %s -name 'general_input'"%(workdir)
a = os.popen(cmd).read().split('\n')[:-1]
if not a:
print 'general_input not found please check and run again'
exit(0)
else:
a = a[0]
with open(a,'r') as FileObj:
for lines in FileObj:
gen = lines.split()
path = a
content = sqlite3.Binary(compressBuf(path))
path = path.replace(
env_var['basedir']+'/','')
#extra_files.append([path, content])
cmd = "find %s -name 'betavalues' -o -name 'sixdesktunes'"%(workdir)
a = os.popen(cmd).read().split('\n')[:-1]
if not a:
print 'betavalues and sixdesktunes files missing'
exit(0)
for dirName in a:
files = dirName.split('/')[-1]
dirName = dirName.replace('/'+files,'')
dirn = dirName.replace(workdir+'/','').split('/')
seed = int(dirn[0])
tunex, tuney = dirn[2].split('_')
if not (seed in rows.keys()):
rows[seed] = []
temp = [seed, tunex, tuney]
if 'betavalues' in files:
f = open(os.path.join(dirName, files), 'r')
beta = [float(i) for i in f.read().split()]
if 'sixdesktunes' in files:
f = open(os.path.join(dirName, files), 'r')
six = [float(i) for i in f.read().split()]
f.close()
if beta and temp and six:
rows[seed].append(temp + beta + gen + six)
beta = temp = six = []
if rows:
lst = dict_to_list(rows)
tab.insertl(lst)
if extra_files:
tab1.insertl(extra_files)
def st_mad6t_run2(self,env_var):
''' store fort.3 and tmp files'''
conn = self.conn
env_var = self.orig_env_var
cols = SQLTable.cols_from_fields(tables.Files.fields)
tab1 = SQLTable(conn,'files',cols,tables.Files.key)
workdir = env_var['sixtrack_input']
extra_files = []
for dirName, _, fileList in os.walk(workdir):
for files in fileList:
if 'fort.3' in files or files.endswith('.tmp'):
path = os.path.join(dirName, files)
content = sqlite3.Binary(compressBuf(path))
path = path.replace(
env_var['scratchdir']+'/','')
extra_files.append([path, content])
if extra_files:
tab1.insertl(extra_files)
def from_dir(cls,studyDir,basedir='.',verbose=False,dryrun=False):
'''create local Database for storing study'''
cls = None
if not (os.path.exists(studyDir+'/sixdeskenv') and \
os.path.exists(studyDir+'/sysenv')):
print "Error in loaddir from %s:"%studyDir,
print "`sixdeskenv` and `sysenv` files should both be present"
exit(0)
env_var = sixdeskdir.parse_env(studyDir)
for key in env_var.keys():
if key not in tables.acc_var:
del env_var[key]
env_var['env_timestamp']=str(time.time())
db = env_var['LHCDescrip'] + ".db"
conn = sqlite3.connect(db, isolation_level="IMMEDIATE")
cur = conn.cursor()
cur.execute("PRAGMA synchronous = OFF")
cur.execute("PRAGMA journal_mode = MEMORY")
cur.execute("PRAGMA auto_vacuum = FULL")
cur.execute("PRAGMA temp_store = MEMORY")
cur.execute("PRAGMA count_changes = OFF")
cur.execute("PRAGMA mmap_size=2335345345")
conn.text_factory=str
cols=SQLTable.cols_from_fields(tables.Env.fields)
tab = SQLTable(conn,'env',cols,tables.Env.key)
temp = tab.select('count(*)')[0][0]
if temp > 0:
print "study found updating..."
lst = tab.select("keyname,value")
cls = SixDeskDB(env_var['LHCDescrip'],basedir,verbose,dryrun)
mtime = os.path.getmtime(studyDir+"/sixdeskenv")
cls.set_variable(lst,mtime)
else:
print "study not found inserting..."
SixDeskDB.st_env(conn,env_var,studyDir)
if cls is None:
cls = SixDeskDB(env_var['LHCDescrip'],basedir,verbose,dryrun)
cls.st_mad6t_run()
cls.st_mad6t_run2(env_var)
cls.st_mad6t_results(env_var)
cls.st_six_beta(env_var)
cls.st_six_input(env_var)
cls.st_six_results(env_var)
return cls
def st_env(conn,env_var,studyDir): '''store environment variables to DB''' extra_files = [] cols=SQLTable.cols_from_fields(tables.Env.fields) tab = SQLTable(conn,'env',cols,tables.Env.key) cols=SQLTable.cols_from_fields(tables.Files.fields) tab1 = SQLTable(conn,'files',cols,tables.Files.key) env_var['env_timestamp']=str(time.time()) mtime = os.path.getmtime(studyDir+"/sixdeskenv") env_var1 = [[i,env_var[i],mtime] for i in env_var.keys()] tab.insertl(env_var1) path = os.path.join(studyDir, 'sixdeskenv') content = sqlite3.Binary(compressBuf(path)) path = 'sixdeskenv' #path = path.replace(env_var['basedir']+'/','') extra_files.append([path, content]) path = os.path.join(studyDir, 'sysenv') content = sqlite3.Binary(compressBuf(path)) path = 'sysenv' #path = path.replace(env_var['basedir']+'/','') extra_files.append([path, content]) tab1.insertl(extra_files)
def st_six_input(self,env_var):
''' store input values (seed,tunes,amps,etc) along with fort.3 file'''
conn = self.conn
cur = conn.cursor()
env_var = self.orig_env_var
cols = SQLTable.cols_from_fields(tables.Six_In.fields)
tab = SQLTable(conn,'six_input',cols,tables.Six_In.key)
#tab = SQLTable(conn,'mad6t_results',cols,tables.Mad_Res.key)
maxtime = tab.select("max(mtime)")[0][0]
count = 0
if not maxtime:
maxtime = 0
cols = SQLTable.cols_from_fields(tables.Files.fields)
tab1 = SQLTable(conn,'files',cols,tables.Files.key)
workdir = os.path.join(env_var['sixdesktrack'],env_var['LHCDescrip'])
extra_files = []
rows = []
six_id = 1
cmd = """find %s -name 'fort.3.gz'"""%(workdir)
a = os.popen(cmd).read().split('\n')[:-1]
print 'fort.3 files =',len(a)
for dirName in a:
files = dirName.split('/')[-1]
dirName = dirName.replace('/'+files,'')
if not ('-' in dirName) \
and (os.path.getmtime(dirName)>maxtime):
mtime = os.path.getmtime(dirName)
dirn = dirName.replace(workdir + '/', '')
dirn = re.split('/|_', dirn)
dirn = [six_id] + dirn
dirn.extend([sqlite3.Binary(open(
os.path.join(dirName, files), 'r'
).read()
),mtime])
rows.append(dirn)
dirn = []
six_id += 1
count += 1
if rows:
tab.insertl(rows)
rows = []
print 'no of fort.3 updated =',count
def dbtocentral(host,user,password,db,localdb,bo=False):
try:
filterwarnings('ignore', category = Warning)
# conn = connect(args,user,password)
# sql = "create database if not exists %s"
# conn.cursor().execute(sql%(a[0]))
conn = connect(host,user,password,db)
if not localdb.endswith('.db'):
localdb += '.db'
if not os.path.isfile(localdb):
print 'db not found'
exit(0)
conn1 = sqlite3.connect(localdb)
conn1.text_factory=str
conn.autocommit(False)
# cur = conn.cursor()
cur1 = conn1.cursor()
# try:
cur = conn.cursor()
# cur.execute("drop table if exists env")
# cur.execute("drop table if exists mad6t_run")
# cur.execute("drop table if exists mad6t_run2")
# cur.execute("drop table if exists mad6t_results")
# cur.execute("drop table if exists six_beta")
# cur.execute("drop table if exists six_input")
# cur.execute("drop table if exists six_results")
# cur.execute("drop table if exists files")
cols=SQLTable.cols_from_fields(tables.Env.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'env',cols,tables.Env.key,"mysql")
sql = "select a.value,b.* from env as a,env as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# cur.executemany("insert into env values(%s,%s,%s)",a)
tab.insertl(a,"%s")
conn.commit()
cols=SQLTable.cols_from_fields(tables.Mad_Run.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'mad6t_run',cols,tables.Mad_Run.key,"mysql")
sql = "select a.value,b.* from env as a,mad6t_run as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
#sql = "insert into mad6t_run values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# print len(a[0])
#cur.executemany(sql,a)
tab.insertl(a,"%s")
conn.commit()
# cols=SQLTable.cols_from_fields(tables.Mad_Run2.fields)
# cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
# cols = ['study VARCHAR(128)'] + cols
# tab = SQLTable(conn,'mad6t_run2',cols)
# sql = "select a.value,b.* from env as a,mad6t_run2 as b where a.keyname = 'LHCDescrip'"
# a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into mad6t_run2 values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
# conn.commit()
cols=SQLTable.cols_from_fields(tables.Mad_Res.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'mad6t_results',cols,tables.Mad_Res.key,"mysql")
sql = "select a.value,b.* from env as a,mad6t_results as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into mad6t_results values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a,"%s")
conn.commit()
cols=SQLTable.cols_from_fields(tables.Six_Be.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'six_beta',cols,tables.Six_Be.key,"mysql")
sql = "select a.value,b.* from env as a,six_beta as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into six_beta values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a,"%s")
conn.commit()
cols=SQLTable.cols_from_fields(tables.Six_In.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'six_input',cols,tables.Six_In.key,"mysql")
sql = "select a.value,b.* from env as a,six_input as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into six_input values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a,"%s")
conn.commit()
cols=SQLTable.cols_from_fields(tables.Six_Res.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'six_results',cols,tables.Six_Res.key,"mysql")
sql = "select a.value,b.* from env as a,six_results as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
maxtime = tab.select("max(mtime)")[0][0]
if not maxtime:
maxtime = 0
a = [list(i) for i in a if i[-1] > maxtime]
# cur.execute("show variables like 'max_%'")
cur = conn.cursor()
cur.execute("set global max_allowed_packet=209715200;")
cur.execute("set global wait_timeout=120;")
cur.execute("set global net_write_timeout=120;")
cur.execute("set global net_read_timeout=120;")
# print list(cur)
# print len(a),(sys.getsizeof(a)/(1024.0))
# sql = "insert into six_results values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
for _ in xrange(len(a)/150000):
# cur.executemany(sql,a[:150000])
# print len(a[:150000][0]),len(cols)
tab.insertl(a[:150000],"%s")
a = a[150000:]
conn.commit()
# cur.executemany(sql,a)
tab.insertl(a,"%s")
conn.commit()
cols=SQLTable.cols_from_fields(tables.Files.fields)
cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn,'files',cols,tables.Files.key,"mysql")
sql = "select a.value,b.* from env as a,files as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into files values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a,"%s")
conn.commit()
if bo:
cmd = "python boinc.py %s"%(localdb)
os.spawnl(os.P_NOWAIT, cmd)
except Error as err:
print("Something went wrong: {}".format(err))
exit(1)
conn.close()
def readplotb(self):
dirname=self.db.mk_analysis_dir()
rectype=[('six_input_id','int'), ('row_num','int'),
('seed','int'), ('qx','float'),('qy','float'),
('betx','float'),('bety','float'),('sigx1','float'),
('sigy1','float'),('deltap','float'),('emitx','float'),
('emity','float'),('sigxminnld', 'float'),
('sigxavgnld' ,'float') ,('sigxmaxnld', 'float'),
('sigyminnld', 'float'),('sigyavgnld' ,'float'),
('sigymaxnld', 'float'),('betx2','float'),
('bety2','float'),('distp','float'),('dist','float'),
('qx_det','float'),('qy_det','float'),('sturns1' ,'int'),
('sturns2','int'),('turn_max','int'),('amp1','float'),
('amp2','float'),('angle','float'),('smearx','float'),
('smeary','float'),('mtime','float')]
names=','.join(zip(*rectype)[0])
turnsl, turnse = self.db.env_var['turnsl'], self.db.env_var['turnse']
tunex, tuney = float(self.db.env_var['tunex']), float(self.db.env_var['tuney'])
ns1l, ns2l = self.db.env_var['ns1l'], self.db.env_var['ns2l']
sql='SELECT %s FROM results ORDER BY tunex,tuney,seed,amp1,amp2,angle'%names
Elhc,Einj=self.db.execute('SELECT emitn,gamma from six_beta LIMIT 1')[0]
anumber=1
seeds, angles= self.db.get_seeds(), self.db.get_angles()
mtime=self.db.execute('SELECT max(mtime) from results')[0][0]
final=[]
post_data=[]
ftot = []
sql1='SELECT %s FROM results WHERE betx>0 AND bety>0 AND emitx>0 AND emity>0 AND turn_max=%d '%(names,turnsl)
nPlotSeeds = self.db.env_var["iend"]
for tunex,tuney in self.db.get_db_tunes():
sixdesktunes="%s_%s"%(tunex,tuney)
sql1+=' AND tunex=%s AND tuney=%s '%(tunex,tuney)
for angle in angles:
fndot='DAres.%s.%s.%s.%d'%(self.db.LHCDescrip,sixdesktunes,turnse,anumber)
fndot=os.path.join(dirname,fndot)
fhdot = open(fndot, 'w')
nSeed=1
for seed in seeds:
name2 = "DAres.%s.%s.%s"%(self.db.LHCDescrip,sixdesktunes,turnse)
name1= '%s%ss%s%s-%s%s.%d'%(self.db.LHCDescrip,seed,sixdesktunes,ns1l, ns2l, turnse,anumber)
ich1, ich2, ich3 = 0, 0, 0
alost1, alost2 = 0., 0.
achaos, achaos1 = 0, 0
icount = 1.
#------------------------readplot-------------------
tl = np.zeros(ntlmax*ntlint+1)
al = np.zeros(ntlmax*ntlint+1)
ichl =np.zeros(ntlmax*ntlint+1)
for i in range(1, ntlmax):
for j in range(0,ntlint):
tl[(i-1)*ntlint+j] = int(round(10**((i-1)+(j-1)/float(ntlint))))
tl[ntlmax*ntlint]=int(round(10**(float(ntlmax))))
#------------------------readplot-------------------
achaos, achaos1 = 0, 0
alost1, alost2 = 0., 0.
ilost=0
itest=1
fac=2.0
fac2=0.1
fac3=0.01
#-------------------- checkInjection-----------------
self.checkInjection()
#-------------------- checkInjection-----------------
sql=sql1+' AND seed=%s '%seed
sql+=' AND angle=%s '%angle
sql+=' ORDER BY amp1 '
inp=np.array(self.db.execute(sql),dtype=rectype)
if len(inp)==0:
msg="all particle lost for angle = %s and seed = %s"
print msg%(angle,seed)
continue
six_id = inp['six_input_id']
row = inp['row_num']
qx = inp['qx']
qy = inp['qy']
betx = inp['betx']
bety = inp['bety']
dist = inp['dist']
distp = inp['distp']
sigx1 = inp['sigx1']
betx2 = inp['betx2']
bety2 = inp['bety2']
emitx = inp['emitx']
emity = inp['emity']
smeary = inp['smeary']
smearx = inp['smearx']
qx_det = inp['qx_det']
qy_det = inp['qy_det']
sigy1 = inp['sigy1']
deltap = inp['deltap']
sturns1 = inp['sturns1']
sturns2 = inp['sturns2']
turn_max = inp['turn_max']
sigxavgnld = inp['sigxavgnld']
sigyavgnld = inp['sigyavgnld']
sigxmaxnld = inp['sigxmaxnld']
sigxminnld = inp['sigxminnld']
sigymaxnld = inp['sigymaxnld']
sigyminnld = inp['sigyminnld']
xidx=(betx>zero) & (emitx>zero)
yidx=(bety>zero) & (emity>zero)
# xidx, yidx = len(betx), len(bety)
sigx1[xidx]=np.sqrt(betx[xidx]*emitx[xidx])
sigy1[yidx]=np.sqrt(bety[yidx]*emity[yidx])
itest = sum(betx>zero)
# itest = len(betx)
iel=itest-1
rat=0
#------------------------read10-------------------
#############################################
# if sigx1[0]>0:
# rat=sigy1[0]**2*betx[0]/(sigx1[0]**2*bety[0])
# if sigx1[0]**2*bety[0]<sigy1[0]**2*betx[0]:
# rat=2
#############################################
#------------------------read10-------------------
if abs(emitx[0]) < epsilon and abs(sigx1[0])>epsilon and bety > epsilon:
rat=sigy1[0]**2*betx[0]/(sigx1[0]**2*bety[0])
if abs(emity[0]) > abs(emitx[0]) or rat > 1e-10:
rat=0
dummy=np.copy(betx)
betx=bety
bety=dummy
dummy=np.copy(betx2)
betx2=bety2
bety2=dummy
dummy=np.copy(sigxminnld)
sigxminnld=np.copy(sigyminnld)
sigyminnld=dummy
dummy=np.copy(sigx1)
sigx1=sigy1
sigy1=dummy
dummy=np.copy(sigxmaxnld)
sigxmaxnld=np.copy(sigymaxnld)
sigymaxnld=dummy
dummy=np.copy(sigxavgnld)
sigxavgnld=sigyavgnld
sigyavgnld=dummy
dummy=np.copy(emitx)
emitx=emity
emity=dummy
#------------------------ratiosEmittances-------------------
sigma=np.sqrt(betx[0]*Elhc/Einj)
if abs(emity[0])>0 and abs(sigx1[0])>0:
if abs(emitx[0])>= epsilon :
eex=emitx[0]
eey=emity[0]
else:
eey=sigy1[0]**2/bety[0]
eex=sigx1[0]**2/betx[0]
rad=np.sqrt(1+eey/eex)/sigma
else:
rad=1
if abs(sigxavgnld[0])>zero and abs(bety[0])>zero and sigma > 0:
if abs(emitx[0]) < zero :
rad1=np.sqrt(1+(sigyavgnld[0]**2*betx[0])/(sigxavgnld[0]**2*bety[0]))/sigma
else:
rad1=(sigyavgnld[0]*np.sqrt(betx[0])-sigxavgnld[0]*np.sqrt(bety2[0]))/(sigxavgnld[0]*np.sqrt(bety[0])-sigyavgnld[0]*np.sqrt(betx2[0]))
rad1=np.sqrt(1+rad1**2)/sigma
else:
rad1 = 1
#------------------------ratiosEmittances-------------------
############################CHAOTIC BOUNDARIES
#------------------------read10-------------------
amin, amax = 1/epsilon, zero
achaosPlot, achaos1Plot = achaos, achaos1
# f30 = open('fort.30.%d.%d' %(nSeed,anumber),'a')
#------------------------read10-------------------
for i in range(0,iel+1):
#------------------------read10-------------------
# if i==0:
# achaos=rad*sigx1[i] #OJO, NOMES PER READ10B
# achaos1 =achaos
#------------------------read10-------------------
#------------------------readplot-------------------
if abs(sigx1[i]) > epsilon and sigx1[i]<amin:
amin = sigx1[i]
if abs(sigx1[i]) > epsilon and sigx1[i]>amax:
amax=sigx1[i]
#------------------------readplot-------------------
if ich1 == 0 and (distp[i] > fac or distp[i] < 1./fac):
ich1 = 1
achaos=rad*sigx1[i]
iin=i
if ich3 == 0 and dist[i] > fac3 :
ich3=1
iend=i
achaos1=rad*sigx1[i]
if ich2 == 0 and (sturns1[i]<turn_max[i] or sturns2[i]<turn_max[i]):
ich2 = 1
alost2 = rad*sigx1[i]
#------------------------readplot-------------------
for j in range(0, ntlmax*ntlint+1):
if (ichl[j] == 0 and int(round(turn_max[i])) >= tl[j]) and ((int(round(sturns1[i])) < tl[j] or int(round(sturns2[i])) < tl[j])):
ichl[j] = 1
al[j-1] = rad*sigx1[i]
#------------------------readplot-------------------
#------------------------read10-------------------
# if i>0:
# achaosPlot, achaos1Plot = achaos, achaos1
# f30.write("%s\t%f %f %f %f %f\n"%( name1[:39],rad*sigx1[i],distp[i],achaosPlot,alost2,rad1*sigxavgnld[i]))
# f30.close()
#------------------------read10-------------------
if iin != -999 and iend == -999 : iend=iel
if iin != -999 and iend > iin :
for i in range(iin,iend+1) :
if(abs(rad*sigx1[i])>zero):
alost1 += rad1 * sigxavgnld[i]/rad/sigx1[i]
if(i!=iend):
icount+=1.
alost1 = alost1/icount
if alost1 >= 1.1 or alost1 <= 0.9: alost1= -1.*alost1
else:
alost1 = 1.0
al = abs(alost1)*al
alost1 = alost1*alost2
if amin == 1/epsilon: amin = zero
amin=amin*rad
amax=amax*rad
al[al==0]=amax
alost3 = turn_max[1]
sturns1[sturns1==zero] = 1
sturns2[sturns2==zero] = 1
alost3 = min(alost3, min(sturns1),min(sturns2))
if(seed<10):
name1+=" "
if(anumber<10):
name1+=" "
#------------------------readplot-------------------
if achaos==0:
f14Flag = 0
achaos=amin
else:
f14Flag = 1
# f14 = open('fort.14.%d.%d' %(nSeed,anumber),'w')
# f14.write('%s %s\n'%(achaos,alost3/fac))
# f14.write('%s %s\n'%(achaos,turn_max[0]*fac))
# f14.close()
if abs(alost1) < epsilon: alost1=amax
if nSeed != (nPlotSeeds +1):
for i in range(0, iel+1):
post_data.append([six_id[i], row[i], rad*sigx1[i], rad1, alost1,
alost2, alost3, achaos, achaos1,
amin, amax, f14Flag, sqlite3.Binary(al)])
# sql=("UPDATE {0} SET {1}={2}, {3}={4}, {5}={6}, {7}={8}, {9}={10},"+
# " {11}={12}, {13}={14}, {15}={16}, {17}={18}, {19}={20}, {21}={22} " +
# " WHERE six_input_id = {23} AND row_num = {24}").format(
# tbl, "rad", (rad*sigx1[i]), "rad1", rad1, "alost1", alost1,
# "alost2", alost2, "alost3", alost3, "achaos", achaos, "achaos1", achaos1,
# "amin", amin,"amax", amax, 'f14', f14Flag, "al", '?', six_id[i], row[i])
# self.db.conn.cursor().execute(sql, (sqlite3.Binary(al),))
#------------------------readplot-------------------
fmt=' %-39s %10.6f %10.6f %10.6f %10.6f %10.6f %10.6f\n'
fhdot.write(fmt%( name1[:39],achaos,achaos1,alost1,alost2,rad*sigx1[0],rad*sigx1[iel]))
final.append([name2, turnsl, tunex, tuney, int(seed),
angle,achaos,achaos1,alost1,alost2,
rad*sigx1[0],rad*sigx1[iel],mtime])
nSeed +=1
anumber+=1
fhdot.close()
cols=SQLTable.cols_from_fields(tables.Da_Post.fields)
# datab=SQLTable(self.db.conn,'da_post',cols,tables.Da_Post.key,recreate=True)
datab=SQLTable(self.db.conn,'da_post',cols)
datab.insertl(final)
cols1 = SQLTable.cols_from_fields(tables.Six_Post.fields)
tab1 = SQLTable(self.db.conn,'six_post',cols1,tables.Six_Post.key)
tab1.insertl(post_data)
sql="""CREATE VIEW IF NOT EXISTS six_post_results
AS SELECT * FROM results INNER JOIN six_post
ON (results.six_input_id=six_post.six_input_id AND results.row_num=six_post.row_num)"""
self.db.execute(sql)
def st_mad6t_results(self,env_var):
''' store fort.2, fort.8, fort.16 files'''
conn = self.conn
env_var = self.orig_env_var
cols = SQLTable.cols_from_fields(tables.Mad_Res.fields)
tab = SQLTable(conn,'mad6t_results',cols,tables.Mad_Res.key)
maxtime = tab.select("max(fort_mtime)")[0][0]
if not maxtime:
maxtime = 0
rows = []
cmd = "find %s -name 'fort.%s*.gz'"
rows = []
a = os.popen(cmd%(env_var['sixtrack_input'],'2')).read().split('\n')[:-1]
b = os.popen(cmd%(env_var['sixtrack_input'],'8')).read().split('\n')[:-1]
c = os.popen(cmd%(env_var['sixtrack_input'],'16')).read().split('\n')[:-1]
print 'fort.2 files =',len(a)
print 'fort.8 files =',len(b)
print 'fort.16 files =',len(c)
up_a = up_b = up_c = 0
for i in a:
if os.path.getmtime(i) > maxtime:
seed = i.split('/')[-1].split('_')[1].replace(".gz","")
row = [seed,sqlite3.Binary(open(i, 'r').read())]
f8 = i.replace("fort.2","fort.8")
mtime = os.path.getmtime(i)
if f8 in b:
row.extend([sqlite3.Binary(open(f8, 'r').read())])
del b[b.index(f8)]
up_b += 1
else:
row.extend([""])
print 'missing file',f8,'inserting null instead'
f16 = i.replace("fort.2","fort.16")
if f16 in c:
row.extend([sqlite3.Binary(open(f16, 'r').read())])
del c[c.index(f16)]
up_c += 1
else:
row.extend([""])
print 'missing file',f16,'inserting null instead'
row.extend([mtime])
rows.append(row)
up_a += 1
for i in b:
if os.path.getmtime(i) > maxtime:
seed = i.split('/')[-1].split('_')[1].replace(".gz","")
print 'missing file',
print '%s inserting null instead'%(i.replace('fort.8','fort.2'))
row = [seed,"",sqlite3.Binary(open(i, 'r').read())]
mtime = os.path.getmtime(i)
f16 = i.replace('fort.8','fort.16')
if f16 in c:
row.extend([sqlite3.Binary(open(f16, 'r').read())])
del c[c.index(f16)]
up_c += 1
else:
row.extend([""])
print 'missing file',f16,'inserting null instead'
row.extend([mtime])
rows.append(row)
up_b += 1
for i in c:
if os.path.getmtime(i) > maxtime:
seed = i.split('/')[-1].split('_')[1].replace(".gz","")
print 'missing file',
print '%s inserting null instead'%(i.replace('fort.16','fort.2'))
print 'missing file',
print '%s inserting null instead'%(i.replace('fort.16','fort.8'))
row = [seed,"","",sqlite3.Binary(open(i, 'r').read())]
mtime = os.path.getmtime(i)
row.extend([mtime])
rows.append(row)
up_c += 1
if rows:
tab.insertl(rows)
rows = {}
print 'no of fort.2 updated =',up_a
print 'no of fort.8 updated =',up_b
print 'no of fort.16 updated =',up_c
def readplotb(self):
dirname = self.db.mk_analysis_dir()
rectype = [('six_input_id', 'int'),
('row_num', 'int'), ('seed', 'int'), ('qx', 'float'),
('qy', 'float'), ('betx', 'float'), ('bety', 'float'),
('sigx1', 'float'), ('sigy1', 'float'), ('deltap', 'float'),
('emitx', 'float'), ('emity', 'float'),
('sigxminnld', 'float'), ('sigxavgnld', 'float'),
('sigxmaxnld', 'float'), ('sigyminnld', 'float'),
('sigyavgnld', 'float'), ('sigymaxnld', 'float'),
('betx2', 'float'), ('bety2', 'float'), ('distp', 'float'),
('dist', 'float'), ('qx_det', 'float'), ('qy_det', 'float'),
('sturns1', 'int'), ('sturns2', 'int'), ('turn_max', 'int'),
('amp1', 'float'), ('amp2', 'float'), ('angle', 'float'),
('smearx', 'float'), ('smeary', 'float'),
('mtime', 'float')]
names = ','.join(zip(*rectype)[0])
turnsl, turnse = self.db.env_var['turnsl'], self.db.env_var['turnse']
tunex, tuney = float(self.db.env_var['tunex']), float(
self.db.env_var['tuney'])
ns1l, ns2l = self.db.env_var['ns1l'], self.db.env_var['ns2l']
sql = 'SELECT %s FROM results ORDER BY tunex,tuney,seed,amp1,amp2,angle' % names
Elhc, Einj = self.db.execute(
'SELECT emitn,gamma from six_beta LIMIT 1')[0]
anumber = 1
seeds, angles = self.db.get_seeds(), self.db.get_angles()
mtime = self.db.execute('SELECT max(mtime) from results')[0][0]
final = []
post_data = []
ftot = []
sql1 = 'SELECT %s FROM results WHERE betx>0 AND bety>0 AND emitx>0 AND emity>0 AND turn_max=%d ' % (
names, turnsl)
nPlotSeeds = self.db.env_var["iend"]
for tunex, tuney in self.db.get_db_tunes():
sixdesktunes = "%s_%s" % (tunex, tuney)
sql1 += ' AND tunex=%s AND tuney=%s ' % (tunex, tuney)
for angle in angles:
fndot = 'DAres.%s.%s.%s.%d' % (self.db.LHCDescrip,
sixdesktunes, turnse, anumber)
fndot = os.path.join(dirname, fndot)
fhdot = open(fndot, 'w')
nSeed = 1
for seed in seeds:
name2 = "DAres.%s.%s.%s" % (self.db.LHCDescrip,
sixdesktunes, turnse)
name1 = '%s%ss%s%s-%s%s.%d' % (self.db.LHCDescrip, seed,
sixdesktunes, ns1l, ns2l,
turnse, anumber)
ich1, ich2, ich3 = 0, 0, 0
alost1, alost2 = 0., 0.
achaos, achaos1 = 0, 0
icount = 1.
#------------------------readplot-------------------
tl = np.zeros(ntlmax * ntlint + 1)
al = np.zeros(ntlmax * ntlint + 1)
ichl = np.zeros(ntlmax * ntlint + 1)
for i in range(1, ntlmax):
for j in range(0, ntlint):
tl[(i - 1) * ntlint + j] = int(
round(10**((i - 1) + (j - 1) / float(ntlint))))
tl[ntlmax * ntlint] = int(round(10**(float(ntlmax))))
#------------------------readplot-------------------
achaos, achaos1 = 0, 0
alost1, alost2 = 0., 0.
ilost = 0
itest = 1
fac = 2.0
fac2 = 0.1
fac3 = 0.01
#-------------------- checkInjection-----------------
self.checkInjection()
#-------------------- checkInjection-----------------
sql = sql1 + ' AND seed=%s ' % seed
sql += ' AND angle=%s ' % angle
sql += ' ORDER BY amp1 '
inp = np.array(self.db.execute(sql), dtype=rectype)
if len(inp) == 0:
msg = "all particle lost for angle = %s and seed = %s"
print msg % (angle, seed)
continue
six_id = inp['six_input_id']
row = inp['row_num']
qx = inp['qx']
qy = inp['qy']
betx = inp['betx']
bety = inp['bety']
dist = inp['dist']
distp = inp['distp']
sigx1 = inp['sigx1']
betx2 = inp['betx2']
bety2 = inp['bety2']
emitx = inp['emitx']
emity = inp['emity']
smeary = inp['smeary']
smearx = inp['smearx']
qx_det = inp['qx_det']
qy_det = inp['qy_det']
sigy1 = inp['sigy1']
deltap = inp['deltap']
sturns1 = inp['sturns1']
sturns2 = inp['sturns2']
turn_max = inp['turn_max']
sigxavgnld = inp['sigxavgnld']
sigyavgnld = inp['sigyavgnld']
sigxmaxnld = inp['sigxmaxnld']
sigxminnld = inp['sigxminnld']
sigymaxnld = inp['sigymaxnld']
sigyminnld = inp['sigyminnld']
xidx = (betx > zero) & (emitx > zero)
yidx = (bety > zero) & (emity > zero)
# xidx, yidx = len(betx), len(bety)
sigx1[xidx] = np.sqrt(betx[xidx] * emitx[xidx])
sigy1[yidx] = np.sqrt(bety[yidx] * emity[yidx])
itest = sum(betx > zero)
# itest = len(betx)
iel = itest - 1
rat = 0
#------------------------read10-------------------
#############################################
# if sigx1[0]>0:
# rat=sigy1[0]**2*betx[0]/(sigx1[0]**2*bety[0])
# if sigx1[0]**2*bety[0]<sigy1[0]**2*betx[0]:
# rat=2
#############################################
#------------------------read10-------------------
if abs(emitx[0]) < epsilon and abs(
sigx1[0]) > epsilon and bety > epsilon:
rat = sigy1[0]**2 * betx[0] / (sigx1[0]**2 * bety[0])
if abs(emity[0]) > abs(emitx[0]) or rat > 1e-10:
rat = 0
dummy = np.copy(betx)
betx = bety
bety = dummy
dummy = np.copy(betx2)
betx2 = bety2
bety2 = dummy
dummy = np.copy(sigxminnld)
sigxminnld = np.copy(sigyminnld)
sigyminnld = dummy
dummy = np.copy(sigx1)
sigx1 = sigy1
sigy1 = dummy
dummy = np.copy(sigxmaxnld)
sigxmaxnld = np.copy(sigymaxnld)
sigymaxnld = dummy
dummy = np.copy(sigxavgnld)
sigxavgnld = sigyavgnld
sigyavgnld = dummy
dummy = np.copy(emitx)
emitx = emity
emity = dummy
#------------------------ratiosEmittances-------------------
sigma = np.sqrt(betx[0] * Elhc / Einj)
if abs(emity[0]) > 0 and abs(sigx1[0]) > 0:
if abs(emitx[0]) >= epsilon:
eex = emitx[0]
eey = emity[0]
else:
eey = sigy1[0]**2 / bety[0]
eex = sigx1[0]**2 / betx[0]
rad = np.sqrt(1 + eey / eex) / sigma
else:
rad = 1
if abs(sigxavgnld[0]) > zero and abs(
bety[0]) > zero and sigma > 0:
if abs(emitx[0]) < zero:
rad1 = np.sqrt(
1 + (sigyavgnld[0]**2 * betx[0]) /
(sigxavgnld[0]**2 * bety[0])) / sigma
else:
rad1 = (sigyavgnld[0] * np.sqrt(betx[0]) -
sigxavgnld[0] * np.sqrt(bety2[0])) / (
sigxavgnld[0] * np.sqrt(bety[0]) -
sigyavgnld[0] * np.sqrt(betx2[0]))
rad1 = np.sqrt(1 + rad1**2) / sigma
else:
rad1 = 1
#------------------------ratiosEmittances-------------------
############################CHAOTIC BOUNDARIES
#------------------------read10-------------------
amin, amax = 1 / epsilon, zero
achaosPlot, achaos1Plot = achaos, achaos1
# f30 = open('fort.30.%d.%d' %(nSeed,anumber),'a')
#------------------------read10-------------------
for i in range(0, iel + 1):
#------------------------read10-------------------
# if i==0:
# achaos=rad*sigx1[i] #OJO, NOMES PER READ10B
# achaos1 =achaos
#------------------------read10-------------------
#------------------------readplot-------------------
if abs(sigx1[i]) > epsilon and sigx1[i] < amin:
amin = sigx1[i]
if abs(sigx1[i]) > epsilon and sigx1[i] > amax:
amax = sigx1[i]
#------------------------readplot-------------------
if ich1 == 0 and (distp[i] > fac
or distp[i] < 1. / fac):
ich1 = 1
achaos = rad * sigx1[i]
iin = i
if ich3 == 0 and dist[i] > fac3:
ich3 = 1
iend = i
achaos1 = rad * sigx1[i]
if ich2 == 0 and (sturns1[i] < turn_max[i]
or sturns2[i] < turn_max[i]):
ich2 = 1
alost2 = rad * sigx1[i]
#------------------------readplot-------------------
for j in range(0, ntlmax * ntlint + 1):
if (ichl[j] == 0
and int(round(turn_max[i])) >= tl[j]) and (
(int(round(sturns1[i])) < tl[j]
or int(round(sturns2[i])) < tl[j])):
ichl[j] = 1
al[j - 1] = rad * sigx1[i]
#------------------------readplot-------------------
#------------------------read10-------------------
# if i>0:
# achaosPlot, achaos1Plot = achaos, achaos1
# f30.write("%s\t%f %f %f %f %f\n"%( name1[:39],rad*sigx1[i],distp[i],achaosPlot,alost2,rad1*sigxavgnld[i]))
# f30.close()
#------------------------read10-------------------
if iin != -999 and iend == -999: iend = iel
if iin != -999 and iend > iin:
for i in range(iin, iend + 1):
if (abs(rad * sigx1[i]) > zero):
alost1 += rad1 * sigxavgnld[i] / rad / sigx1[i]
if (i != iend):
icount += 1.
alost1 = alost1 / icount
if alost1 >= 1.1 or alost1 <= 0.9:
alost1 = -1. * alost1
else:
alost1 = 1.0
al = abs(alost1) * al
alost1 = alost1 * alost2
if amin == 1 / epsilon: amin = zero
amin = amin * rad
amax = amax * rad
al[al == 0] = amax
alost3 = turn_max[1]
sturns1[sturns1 == zero] = 1
sturns2[sturns2 == zero] = 1
alost3 = min(alost3, min(sturns1), min(sturns2))
if (seed < 10):
name1 += " "
if (anumber < 10):
name1 += " "
#------------------------readplot-------------------
if achaos == 0:
f14Flag = 0
achaos = amin
else:
f14Flag = 1
# f14 = open('fort.14.%d.%d' %(nSeed,anumber),'w')
# f14.write('%s %s\n'%(achaos,alost3/fac))
# f14.write('%s %s\n'%(achaos,turn_max[0]*fac))
# f14.close()
if abs(alost1) < epsilon: alost1 = amax
if nSeed != (nPlotSeeds + 1):
for i in range(0, iel + 1):
post_data.append([
six_id[i], row[i], rad * sigx1[i], rad1,
alost1, alost2, alost3, achaos, achaos1, amin,
amax, f14Flag,
sqlite3.Binary(al)
])
# sql=("UPDATE {0} SET {1}={2}, {3}={4}, {5}={6}, {7}={8}, {9}={10},"+
# " {11}={12}, {13}={14}, {15}={16}, {17}={18}, {19}={20}, {21}={22} " +
# " WHERE six_input_id = {23} AND row_num = {24}").format(
# tbl, "rad", (rad*sigx1[i]), "rad1", rad1, "alost1", alost1,
# "alost2", alost2, "alost3", alost3, "achaos", achaos, "achaos1", achaos1,
# "amin", amin,"amax", amax, 'f14', f14Flag, "al", '?', six_id[i], row[i])
# self.db.conn.cursor().execute(sql, (sqlite3.Binary(al),))
#------------------------readplot-------------------
fmt = ' %-39s %10.6f %10.6f %10.6f %10.6f %10.6f %10.6f\n'
fhdot.write(fmt %
(name1[:39], achaos, achaos1, alost1, alost2,
rad * sigx1[0], rad * sigx1[iel]))
final.append([
name2, turnsl, tunex, tuney,
int(seed), angle, achaos, achaos1, alost1, alost2,
rad * sigx1[0], rad * sigx1[iel], mtime
])
nSeed += 1
anumber += 1
fhdot.close()
cols = SQLTable.cols_from_fields(tables.Da_Post.fields)
# datab=SQLTable(self.db.conn,'da_post',cols,tables.Da_Post.key,recreate=True)
datab = SQLTable(self.db.conn, 'da_post', cols)
datab.insertl(final)
cols1 = SQLTable.cols_from_fields(tables.Six_Post.fields)
tab1 = SQLTable(self.db.conn, 'six_post', cols1, tables.Six_Post.key)
tab1.insertl(post_data)
sql = """CREATE VIEW IF NOT EXISTS six_post_results
AS SELECT * FROM results INNER JOIN six_post
ON (results.six_input_id=six_post.six_input_id AND results.row_num=six_post.row_num)"""
self.db.execute(sql)
def make_result_table(self):
jobnames = [c[0] for c in JobParams.fields]
res = JobParams.fields + Fort10.fields
cols = SQLTable.cols_from_fields(res)
self.results = SQLTable(self.db, 'results', cols, keys=jobnames)
def make_result_table(self): jobnames=[c[0] for c in JobParams.fields] res=JobParams.fields+Fort10.fields cols=SQLTable.cols_from_fields(res) self.results=SQLTable(self.db,'results',cols,keys=jobnames)
def dbtocentral(host, user, password, db, localdb, bo=False):
try:
filterwarnings('ignore', category=Warning)
# conn = connect(args,user,password)
# sql = "create database if not exists %s"
# conn.cursor().execute(sql%(a[0]))
conn = connect(host, user, password, db)
if not localdb.endswith('.db'):
localdb += '.db'
if not os.path.isfile(localdb):
print 'db not found'
exit(0)
conn1 = sqlite3.connect(localdb)
conn1.text_factory = str
conn.autocommit(False)
# cur = conn.cursor()
cur1 = conn1.cursor()
# try:
cur = conn.cursor()
# cur.execute("drop table if exists env")
# cur.execute("drop table if exists mad6t_run")
# cur.execute("drop table if exists mad6t_run2")
# cur.execute("drop table if exists mad6t_results")
# cur.execute("drop table if exists six_beta")
# cur.execute("drop table if exists six_input")
# cur.execute("drop table if exists six_results")
# cur.execute("drop table if exists files")
cols = SQLTable.cols_from_fields(tables.Env.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'env', cols, tables.Env.key, "mysql")
sql = "select a.value,b.* from env as a,env as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# cur.executemany("insert into env values(%s,%s,%s)",a)
tab.insertl(a, "%s")
conn.commit()
cols = SQLTable.cols_from_fields(tables.Mad_Run.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'mad6t_run', cols, tables.Mad_Run.key, "mysql")
sql = "select a.value,b.* from env as a,mad6t_run as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
#sql = "insert into mad6t_run values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# print len(a[0])
#cur.executemany(sql,a)
tab.insertl(a, "%s")
conn.commit()
# cols=SQLTable.cols_from_fields(tables.Mad_Run2.fields)
# cols = [i.replace("STRING","VARCHAR(128)") for i in cols]
# cols = ['study VARCHAR(128)'] + cols
# tab = SQLTable(conn,'mad6t_run2',cols)
# sql = "select a.value,b.* from env as a,mad6t_run2 as b where a.keyname = 'LHCDescrip'"
# a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into mad6t_run2 values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
# conn.commit()
cols = SQLTable.cols_from_fields(tables.Mad_Res.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'mad6t_results', cols, tables.Mad_Res.key,
"mysql")
sql = "select a.value,b.* from env as a,mad6t_results as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into mad6t_results values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a, "%s")
conn.commit()
cols = SQLTable.cols_from_fields(tables.Six_Be.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'six_beta', cols, tables.Six_Be.key, "mysql")
sql = "select a.value,b.* from env as a,six_beta as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into six_beta values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a, "%s")
conn.commit()
cols = SQLTable.cols_from_fields(tables.Six_In.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'six_input', cols, tables.Six_In.key, "mysql")
sql = "select a.value,b.* from env as a,six_input as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into six_input values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a, "%s")
conn.commit()
cols = SQLTable.cols_from_fields(tables.Six_Res.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'six_results', cols, tables.Six_Res.key, "mysql")
sql = "select a.value,b.* from env as a,six_results as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
maxtime = tab.select("max(mtime)")[0][0]
if not maxtime:
maxtime = 0
a = [list(i) for i in a if i[-1] > maxtime]
# cur.execute("show variables like 'max_%'")
cur = conn.cursor()
cur.execute("set global max_allowed_packet=209715200;")
cur.execute("set global wait_timeout=120;")
cur.execute("set global net_write_timeout=120;")
cur.execute("set global net_read_timeout=120;")
# print list(cur)
# print len(a),(sys.getsizeof(a)/(1024.0))
# sql = "insert into six_results values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
for _ in xrange(len(a) / 150000):
# cur.executemany(sql,a[:150000])
# print len(a[:150000][0]),len(cols)
tab.insertl(a[:150000], "%s")
a = a[150000:]
conn.commit()
# cur.executemany(sql,a)
tab.insertl(a, "%s")
conn.commit()
cols = SQLTable.cols_from_fields(tables.Files.fields)
cols = [i.replace("STRING", "VARCHAR(128)") for i in cols]
cols = ['study VARCHAR(128)'] + cols
tab = SQLTable(conn, 'files', cols, tables.Files.key, "mysql")
sql = "select a.value,b.* from env as a,files as b where a.keyname = 'LHCDescrip'"
a = [list(i) for i in list(cur1.execute(sql))]
# sql = "insert into files values (%s)"%(','.join("%s " for _ in xrange(len(cols))))
# cur.executemany(sql,a)
tab.insertl(a, "%s")
conn.commit()
if bo:
cmd = "python boinc.py %s" % (localdb)
os.spawnl(os.P_NOWAIT, cmd)
except Error as err:
print("Something went wrong: {}".format(err))
exit(1)
conn.close()