def runMainTest(param) :
# SQL connection parameters for the test and result databases
resultDBParam = {'dbname' : param.get('resultdbname'), 'host' : param.get('resulthost'), 'port' : param.get('resultport'),'user' : param.get('resultuser') \
'password' : param.get('resultuserpassword')}
testDBParam = {'dbname' : param.get('testdbname'), 'host' : param.get('testhost'), 'port' : param.get('testport'),'user' : param.get('testuser') \
'password' : param.get('testuserpassword')}
script = tests.formulateTestQuery(param['testtype']) if param['testtype'] != 'custom' else param.get('customfile')
resultDBParam.update( 'dbname' : 'postgres' )
resultDBExists = sql.queryDB (resultDBParam,uf.checkDBExist(param.get('resultdbname')) ,'read')[0][1]
if resultDBExists != param.get('resultdbname') :
sql.queryDB (resultDBParam,uf.createDBText(param.get('resultdbname')) ,'write')
resultDBParam.update( 'dbname' : param.get('resultdbname') )
result = sql.queryDB (resultDBParam,init.getInitdbText() ,'write')[0][1]
if result != 0 :
print ('Could not create resultdb objects')
sys.exit(0)
else :
resultDBParam.update( 'dbname' : param.get('resultdbname') )
# obtain disk mount points
devices = param.get('tablespaces')
for tblspace in devices :
testDBParam.update( 'dbname' : 'postgres' )
sql.queryDB (testDBParam,uf.dropDBText(param.get('testdbname')) ,'write')
sql.queryDB (testDBParam,uf.dropTableSpaceText() ,'write')
sql.queryDB (testDBParam,uf.createTableSpaceText(tblspace) ,'write')
sql.queryDB (testDBParam,uf.createDBText(param.get('testdbname')) + ' TABLESPACE ' + tblspace ,'write')
testDBParam.update( 'dbname' : param.get('testdbname') )
pgversion = uf.getDBVersion(testDBParam)
dataDirLocation = tblspace
if tblspace == 'pg_default' :
dataDirLocation = uf.getCurrentDBSetting(testDBParam,'data_directory')
# create the main test entry
sysinfo = sql.queryDB (testDBParam,uf.getSysInfo(pgversion),'read')
testset = sql.queryDB (resultDBParam,uf.insertNewTest(pgversion,sysinfo,dataDirLocation),'read')
# if a custom script is used, set scale to 1
scales = uf.getLevelScale( param.get('testtype') )
for key in scales.keys() :
if param.get('initdb') == 0 :
sql.queryDB (testDBParam,uf.droppgBenchTables(),'write')
sql.queryDB (testDBParam,'VACUUM FULL','write')
print ('Creating new pgbench tables')
if pgversion >= 9.4 :
subprocess.check_output(uf.utilfunc('testdb','pgbench',param) + ['-i','-s',str(scales.get(key)),'--foreign-keys'])
else :
subprocess.check_output(uf.utilfunc('testdb','pgbench',param) + ['-i','-s',str(scales.get(key))])
# running the main test
if pgversion >= 8.4 :
runtime = param.get('runtime')
if trans != None :
for trans in param.get('transactions') :
testComponents(param.get('repeattime'), param.get('clients') , testset,scales.get(key),script,param,dataDirLocation,trans,runtime)
else :
testComponents(param.get('repeattime'), param.get('clients') , testset,scales.get(key),script,param,dataDirLocation,runtime)
else :
print ("PostgreSQL version number not supported .... \n supported version >=8.4")
sys.exit(0)
tests.deleteFiles(param['testtype'] + '.sql')
tests.deleteFiles('timing.csv')
tests.deleteFiles('result.txt')
tests.deleteFiles('*_log*')
tests.deleteFiles('load.csv')
tests.deleteFiles('disk_io_count.csv')
def runBenchwarmer (testset,repeat,scale,client,script,param,datadir,trans=None,runtime) :
# pgbench -t 1000 -T 6000 -M simple -l -j 4 -c 50 dbname
queryMode = param.get('querymode')
thread = param.get('threadcount')
db = param.get('testdbname')
# this will throw a huge error if thread is not a multiple of client, so we try to normilize that
if int(client) % int(thread) != 0 :
client = str( int(thread) * ( int(client) / int(thread) ) )
out = ''
start_time = str(datetime.datetime.now()) # might be a few milli or micro sec off, but it won't influence the results.
end_time = ''
# start saving the cpu load and the I/O count
cpu_load = subprocess.Popen( [ cpu.(5,str(testset),str(repeat),str(scale)) ] , stdout=subprocess.PIPE )
io_count = subprocess.Popen( [ io.getIOCount(5, datadir, str(testset), str(repeat), str(scale)) ] , stdout=subprocess.PIPE )
if trans == None :
out = subprocess.check_output( uf.utilfunc('testdb','pgbench',param) + ['-M',queryMode,'-f',script,'-T',runtime,'-j',\
thread,'-c',client,'-l','-s',str(scale),db] )
end_time = str(datetime.datetime.now())
elif :
out = subprocess.check_output( uf.utilfunc('testdb','pgbench',param) + ['-M',queryMode,'-f',script,'-t',trans,'-j',\
thread,'-c',client,'-l','-s',str(scale),db] )
end_time = str(datetime.datetime.now())
else :
print ("Either the runtime or transaction/client must be specified ...")
cpu_load.stdout.close()
cpu_load.terminate()
io_count.stdout.close()
io_count.terminate()
sys.exit(1)
cpu_load.stdout.close()
cpu_load.terminate()
io_count.stdout.close()
io_count.terminate()
f = open('result.txt','wb')
f.write(out)
f.close()
print (out)
r = open('result.txt','r')
trans = ''
tps = ''
for lines in r :
if lines.startswith('number of transactions actually processed:') == True :
trans = lines.split()[-1]
elif lines.endswith('(including connections establishing)\n') == True :
tps = lines.split()[2]
r.close()
test = subprocess.check_output(uf.utilfunc('resultdb','psql',param) + ['-tAc',\
uf.insertTestResult( script,client,thread,scale,param.get('testdbname'),start_time.rstrip(),end_time.rstrip(),tps,trans )] )
for f in glob.glob('*_log*') :
uf.writeCSV(f,test.split("\n")[0])