def __executeCaseFile(self, file_name, run_id, restart=False):
"""Execute case file with specific greenplum/postgres configuration.
param file_name, file name of case
param run_id, the index of this run
param restart, dose the test need to restart greenplum
"""
connection_str = self.dbManager.getDBConnection()
exe_append = " -C " + connection_str \
+ " --run_id " + str(run_id) \
+ " --analyticstool " + self.platform
lines = open(file_name).readlines()
for line in lines:
line = line.strip()
if line and line[0] != '#':
if line.startswith('psql'):
#this is a psql command, need to add connection options
line = line.replace("psql", "", 1)
run_sql.runSQL(line,
psqlArgs=self.psql_append,
source_path=self.cur_dbconf['env'])
else:
#this is a executor command, need to add run_id and connection options
if restart:
self.dbManager.stop()
self.dbManager.start()
subprocess.call(line + exe_append, shell=True)
def executeCase(self, test_case_path, run_id, restart=False):
"""Execute case files with specific greenplum/postgres configuration
param test_case_path, path of all test case
param run_id, the index of this run
param restart, dose the test need to restart greenplum
"""
self.dbManager.start()
run_sql.runSQL(self.init_sql,
psqlArgs=self.psql_append,
source_path=self.cur_dbconf['env'])
self.__executeCaseWithTool(test_case_path, run_id, restart)
version_sql = 'select madlib.version();'
output = run_sql.runSQL(version_sql,
psqlArgs=self.psql_append,
source_path=self.cur_dbconf['env'])
try:
temp_list = output.split(',')
self.version = temp_list[2]
except Exception:
self.version = "WRONG"
self.dbManager.stop()
def __executeCaseFile(self, file_name, run_id, restart = False):
"""Execute case file with specific greenplum/postgres configuration.
param file_name, file name of case
param run_id, the index of this run
param restart, dose the test need to restart greenplum
"""
connection_str = self.dbManager.getDBConnection()
exe_append = " -C " + connection_str \
+ " --run_id " + str(run_id) \
+ " --analyticstool " + self.platform
lines = open(file_name).readlines()
for line in lines:
line = line.strip()
if line and line[0] != '#':
if line.startswith('psql'):
#this is a psql command, need to add connection options
line = line.replace("psql", "", 1)
run_sql.runSQL(line, psqlArgs = self.psql_append, source_path = self.cur_dbconf['env'])
else:
#this is a executor command, need to add run_id and connection options
if restart:
self.dbManager.stop()
self.dbManager.start()
subprocess.call(line + exe_append, shell = True)
def gen_failedcasesreport(self):
FailedCasesreport_sql = "SELECT DISTINCT casename FROM %s.failedcases;"% self.schema
FailedCasesreport_filename = self.ReportDir + self.platform + '_' + self.run_id + '_FailedCases.report'
copyToFile(self.psql_args, FailedCasesreport_sql, os.path.join(self.CWD, FailedCasesreport_filename))
dirpath = self.ReportDir + self.platform + '_' + self.run_id + "Failedcases_report" + '/'
os.system('rm -rf ' + dirpath)
os.system('mkdir -p ' + dirpath)
sql="select %s.gen_failedreport( '%s', '%s')"%(self.schema, dirpath, self.schema)
try:
run_sql.runSQL(sql, psqlArgs = self.psql_args)
except Exception,exp:
print exp
print '\nError when generating failed cases reports'
def run(self):
"""Run the madlib invoking sql statement, and store the result to logger database."""
is_successful = False
sql = self.sql
# print executor command and invocation sql statement
print "----------------------------------------------------"
print " ".join(self.argv)
print self.sql
timer = run_sql.Timer()
# the elapsed time is in second, but we will convert it to ms in database
with timer:
try:
result = run_sql.runSQL(
sql,
self.logusername,
self.logpassword,
self.loghostname,
self.logport,
self.logdatabase,
["--expanded"],
)
is_successful = True
except Exception as e:
result = str(e)
print str(e)
try:
# strip empty line, and use only the tail of 1k char
result = "\n".join([l for l in result.splitlines() if l.strip() != ""])
result = result[-1000:]
self.logger.log_test_item(timer.elapsed, is_successful, "table", result, sql, "false")
if is_successful:
self.__store_db_result(result)
except Exception as e:
print "ERROR", str(e)
def run(self):
"""Run the madlib invoking sql statement, and store the result to logger database."""
is_successful = False
sql = self.sql
#print executor command and invocation sql statement
print '----------------------------------------------------'
print ' '.join(self.argv)
print self.sql
timer = run_sql.Timer()
#the elapsed time is in second, but we will convert it to ms in database
with timer:
try:
result = run_sql.runSQL(sql, self.logusername,
self.logpassword, self.loghostname,
self.logport, self.logdatabase,
['--expanded'])
is_successful = True
except Exception as e:
result = str(e)
print str(e)
try:
#strip empty line, and use only the tail of 1k char
result = '\n'.join(
[l for l in result.splitlines() if l.strip() != ''])
result = result[-1000:]
self.logger.log_test_item(timer.elapsed, is_successful, "table", \
result, sql, "false")
if is_successful:
self.__store_db_result(result)
except Exception as e:
print 'ERROR', str(e)
def main():
psql_args= ['-U', 'gpdbchina', '-h', 'localhost', '-p', '5466', '-d', 'benchreport']
ret = run_sql.runSQL("select max(runid) from benchmark.testitemseq;", psqlArgs = psql_args)
run_id = ret.splitlines()[2].strip()
platform = "TEST"
schema = "benchmark"
generate_report(psql_args, schema, run_id, platform)
def copyToFile(psql_args, sql, filename):
"""generate test result report
param sql as select, but we need to create as temp table to store
those data and copy to file
"""
out = run_sql.runSQL(sql, psqlArgs = psql_args)
f = open(filename, 'w')
f.write(out)
f.close()
def executeCase(self, test_case_path, run_id, restart=False):
"""Execute case files with specific greenplum/postgres configuration
param test_case_path, path of all test case
param run_id, the index of this run
param restart, dose the test need to restart greenplum
"""
self.dbManager.start()
run_sql.runSQL(self.init_sql, psqlArgs=self.psql_append, source_path=self.cur_dbconf["env"])
self.__executeCaseWithTool(test_case_path, run_id, restart)
version_sql = "select madlib.version();"
output = run_sql.runSQL(version_sql, psqlArgs=self.psql_append, source_path=self.cur_dbconf["env"])
try:
temp_list = output.split(",")
self.version = temp_list[2]
except Exception:
self.version = "WRONG"
def __load(self, db_manager, kind, yaml_list, overload=False):
"""Load table into db."""
fail_list = []
for yaml_path in yaml_list:
if os.path.exists(os.path.join(self.__yamlPath, yaml_path)) \
is False:
continue
yaml_content = yaml.load(open(os.path.join(self.__yamlPath, yaml_path)))
if 'tables' in yaml_content:
for table in yaml_content['tables']:
if 'skip' in table and \
(table['skip'] == 'all' or (table['skip'] in kind)):
continue
table_name = '.'.join([self.__schema, table['id']])
outSQL = os.path.join(self.__yamlPath, 'sql', table['id'] + '.sql')
output = run_sql.runSQL("SELECT count(*) FROM %s" % table_name,
psqlArgs=db_manager.getDBsqlArgs(),
onErrorStop=False, Return="all")
#If table exists and no nedd to overload, skip this sql.
if output.find('not exist') < 0 and \
output.find(' 0') < 0 and overload is False:
continue
elif output.find(' 0') > 0 and overload is False:
fail_list.append(table['id'])
print "ERROR : Success create but copy failed : %s" % table['id']
continue
try:
start = time.time()
subprocess.check_call('gunzip -f %s.gz' % outSQL, shell=True)
run_sql.runSQL(outSQL, logport=db_manager.db_conf['port'],
logdatabase=db_manager.db_conf['database'],
onErrorStop=False, isFile=True,
source_path=db_manager.getDBenv())
subprocess.check_call('gzip -f %s' % outSQL, shell=True)
self.__logInfo(table['id'], 'load', time.time() - start)
#Load additional sql file for table.
if 'sql' in table:
run_sql.runSQL(os.path.join(self.__yamlPath,
os.path.dirname(yaml_path), table['sql']),
logport=db_manager.db_conf['port'],
logdatabase=db_manager.db_conf['database'],
onErrorStop=False, isFile=True,
source_path=db_manager.getDBenv())
print "INFO : Success Loaded : %s " % table['id']
except:
fail_list.append(table['id'])
print "ERROR : Fail Loaded : %s " % table['id']
#Load additional sql file for algorithm.
if 'sql' in yaml_content:
run_sql.runSQL(os.path.join(self.__yamlPath,
os.path.dirname(yaml_path), yaml_content['sql']),
logport=db_manager.db_conf['port'],
logdatabase=db_manager.db_conf['database'],
onErrorStop=False, isFile=True,
source_path=db_manager.getDBenv())
print "FAILED LOAD TABLES:\n", fail_list
def runCases(getfile, skipfile, isList, isUnique, platform, analyticsTool,
testConfiger, run_id):
"""run the cases specialized by getfile. """
(cases, skippedcases) = __parserCasesFromFile(getfile, skipfile, isList,
isUnique, testConfiger)
if not platform in analyticsTool.analyticsTools:
sys.exit('ERROR: Wrong platform name.')
testdbconf = analyticsTool.analyticsTools[platform]
executor = execute_case.TestCaseExecutor(cases, testdbconf, platform)
executor.executeCase(Path.TestCaseDir, run_id)
if skipfile:
for case in skippedcases:
f = open(Path.TestCaseDir + case + '.case')
lines = f.readlines()
for line in lines:
if len(line) < 10:
continue
pos = line.find('target_base_name')
if pos > 0:
target_base_name = line[pos + 17:].strip()
else:
continue
print target_base_name
pos = line.find('-c')
cmd = line[pos + 3:].strip()
sql = """insert into %s.testitemresult
values( '%s', %s, %s, '%s',
'%s', %s, %s, '%s', '%s', %s::bool);
""" % (testConfiger.getResultSchema(), target_base_name, run_id, \
0, 'table', platform, 0,
'NULL', 'NULL', cmd, False)
result = run_sql.runSQL(sql, testConfiger.user, None,
testConfiger.host, testConfiger.port,
testConfiger.database, ['--expanded'])
return executor.version
def runCases(getfile, skipfile, isList, isUnique, platform, analyticsTool, testConfiger, run_id):
"""run the cases specialized by getfile. """
(cases, skippedcases) = __parserCasesFromFile(getfile, skipfile, isList, isUnique, testConfiger)
if not platform in analyticsTool.analyticsTools:
sys.exit('ERROR: Wrong platform name.')
testdbconf = analyticsTool.analyticsTools[platform]
executor = execute_case.TestCaseExecutor(cases, testdbconf, platform)
executor.executeCase(Path.TestCaseDir, run_id)
if skipfile:
for case in skippedcases:
f = open( Path.TestCaseDir +case+'.case')
lines = f.readlines()
for line in lines:
if len(line) < 10:
continue
pos =line.find('target_base_name')
if pos > 0:
target_base_name = line[pos + 17:].strip()
else:
continue
print target_base_name
pos = line.find('-c')
cmd = line[pos + 3:].strip()
sql = """insert into %s.testitemresult
values( '%s', %s, %s, '%s',
'%s', %s, %s, '%s', '%s', %s::bool);
""" % (testConfiger.getResultSchema(), target_base_name, run_id, \
0, 'table', platform, 0,
'NULL', 'NULL', cmd, False)
result = run_sql.runSQL(sql, testConfiger.user, None, testConfiger.host, testConfiger.port, testConfiger.database,['--expanded'])
return executor.version
def __load(self, db_manager, kind, yaml_list, overload=False):
"""Load table into db."""
fail_list = []
for yaml_path in yaml_list:
if os.path.exists(os.path.join(self.__yamlPath, yaml_path)) is False:
continue
yaml_content = yaml.load(open(os.path.join(self.__yamlPath, yaml_path)))
if "tables" in yaml_content:
for table in yaml_content["tables"]:
if "skip" in table and (table["skip"] == "all" or (table["skip"] in kind)):
continue
table_name = ".".join([self.__schema, table["id"]])
outSQL = os.path.join(self.__yamlPath, "sql", table["id"] + ".sql")
output = run_sql.runSQL(
"SELECT count(*) FROM %s" % table_name,
psqlArgs=db_manager.getDBsqlArgs(),
onErrorStop=False,
Return="all",
)
# If table exists and no nedd to overload, skip this sql.
if output.find("not exist") < 0 and output.find(" 0") < 0 and overload is False:
continue
elif output.find(" 0") > 0 and overload is False:
fail_list.append(table["id"])
print "ERROR : Success create but copy failed : %s" % table["id"]
continue
try:
start = time.time()
subprocess.check_call("gunzip -f %s.gz" % outSQL, shell=True)
run_sql.runSQL(
outSQL,
logport=db_manager.db_conf["port"],
logdatabase=db_manager.db_conf["database"],
onErrorStop=False,
isFile=True,
source_path=db_manager.getDBenv(),
)
subprocess.check_call("gzip -f %s" % outSQL, shell=True)
self.__logInfo(table["id"], "load", time.time() - start)
# Load additional sql file for table.
if "sql" in table:
run_sql.runSQL(
os.path.join(self.__yamlPath, os.path.dirname(yaml_path), table["sql"]),
logport=db_manager.db_conf["port"],
logdatabase=db_manager.db_conf["database"],
onErrorStop=False,
isFile=True,
source_path=db_manager.getDBenv(),
)
print "INFO : Success Loaded : %s " % table["id"]
except:
fail_list.append(table["id"])
print "ERROR : Fail Loaded : %s " % table["id"]
# Load additional sql file for algorithm.
if "sql" in yaml_content:
run_sql.runSQL(
os.path.join(self.__yamlPath, os.path.dirname(yaml_path), yaml_content["sql"]),
logport=db_manager.db_conf["port"],
logdatabase=db_manager.db_conf["database"],
onErrorStop=False,
isFile=True,
source_path=db_manager.getDBenv(),
)
print "FAILED LOAD TABLES:\n", fail_list
def main():
"""Change the options, del -c -S -t -r. Modify -s. i.e. python run.py -s map.yaml"""
use = """usage: ./run.py --loaddata --gencase --init
--schedule map_file
--genreport run_id
--init or -i for short, to clean up and init logger database
--schedule or -s for short, to load the schedule file and run
--loaddata or -l for short, to load data or not
--gencase or -g for short, to generate test cases
--genreport or -G for short, to generate test report by run_id
"""
parser = ArgumentParser(description=use)
parser.add_argument("-i", "--initbenchmark", action='store_true', help = "Initial benchmark db.")
parser.add_argument("-g", "--gencase", action='store_true', help = "Generate cases.")
parser.add_argument("-d", "--debug", action = 'store_true', help ="Debug model will generate all sql for each case, it take a long time.")
parser.add_argument("-l", "--forceload", action='store_true', help = "Reconvert and reload all tables set in config.yaml and tables.yaml.")
parser.add_argument("-L", "--smartload", action='store_true', help = "Load modules by config.yaml, tables.yaml and -m. If table exists in db, do nothing.")
parser.add_argument("-m", "--module", nargs = "*", help = "Modules selected to load. For exapme: 'run.py -Lm Dec Ran' means only load decisition tree and random forest.")
parser.add_argument("-s", "--schedule", help = "Set schedule file and run.")
options = parser.parse_args()
testConfiger = test_config.Configer(Path.TestConfigXml)
testConfiger.testconfig()
analyticsTools = analytics_tool.AnalyticsTools(Path.AnalyticsToolXml)
analyticsTools.parseTools()
psql_args = testConfiger.getResultDBmanager().getDBsqlArgs()
schema = testConfiger.getResultSchema()
tools.set_search_path(schema, Path.BootstrapDir)
if options.schedule:
map_file = options.schedule
plans = tools.parserMap(Path.ScheduleDir + map_file)
run_sql.runSQL("update %s.testitemseq set runid = runid + 1;"%schema, psqlArgs = psql_args)
ret = run_sql.runSQL("select max(runid) from %s.testitemseq;"%schema, psqlArgs = psql_args)
run_id = ret.splitlines()[2].strip()
for plan in plans:
if len(plan) > 4:
sys.exit('ERROR:-s arg file has some grammer error, too many lines.')
if not 'skip' in plan:
plan['skip'] = ""
if 'cases' in plan:
isList = False
filename = plan['cases']
if 'lists' in plan:
isList = True
filename = plan['lists']
if 'unique' in plan and plan['unique']:
isUnique = True
else:
isUnique = False
version = run_case.runCases(filename, plan['skip'], isList, isUnique, plan['platform'], analyticsTools, testConfiger, run_id)
if plan['skip']:
run_sql.runSQL(Path.BootstrapDir + 'skipsqlfile.sql', psqlArgs = psql_args, isFile = True)
run_sql.runSQL(Path.BootstrapDir + 'post.sql', psqlArgs = psql_args, onErrorStop = False, isFile = True)
gen_report.generate_report(psql_args, schema, run_id, plan['platform'], Path.ReportDir)
#load data set to all databases to test
if options.forceload:
loading_manager = loadingManager(Path.RootPath, 'madlibtestdata', analyticsTools)
loading_manager.do(options.module, False, True, True)
if options.initbenchmark:
#initialization
run_sql.runSQL(Path.BootstrapDir + 'init.sql', psqlArgs = psql_args, isFile = True)
run_sql.runSQL(Path.BootstrapDir + 'init_cases.sql', psqlArgs = psql_args, isFile = True)
run_sql.runSQL(Path.BootstrapDir + 'resultbaseline.sql', psqlArgs = psql_args, isFile = True)
#generate new cases
os.system('cd ../src/generator/ && python ./gen_testcase.py')
run_sql.runSQL(Path.BootstrapDir + 'analyticstool.sql', psqlArgs = psql_args, isFile = True)
#initialize algorithm result table
run_sql.runSQL(Path.BootstrapDir + 'algorithmspec.sql', psqlArgs = psql_args, isFile = True)
for sqlfile in glob.glob('../testcase/*.sql'):
run_sql.runSQL(sqlfile, psqlArgs = psql_args, onErrorStop = False, isFile = True)
if options.gencase:
#initialization
run_sql.runSQL(Path.BootstrapDir + 'init_cases.sql', psqlArgs = psql_args, isFile = True)
run_sql.runSQL(Path.BootstrapDir + 'resultbaseline.sql', psqlArgs = psql_args, isFile = True)
#generate new cases
if options.debug:
os.system('cd ../src/generator/ && python ./gen_testcase.py debug')
else:
os.system('cd ../src/generator/ && python ./gen_testcase.py')
run_sql.runSQL(Path.BootstrapDir + 'analyticstool.sql', psqlArgs = psql_args, isFile = True)
run_sql.runSQL(Path.BootstrapDir + 'algorithmspec.sql', psqlArgs = psql_args, onErrorStop = False, isFile = True)
for sqlfile in glob.glob('../testcase/*.sql'):
run_sql.runSQL(sqlfile, psqlArgs = psql_args, onErrorStop = False, isFile = True)
if options.smartload:
loading_manager = loadingManager(Path.RootPath, 'madlibtestdata', analyticsTools)
loading_manager.do(options.module, False, False, False)
def main():
"""Change the options, del -c -S -t -r. Modify -s. i.e. python run.py -s map.yaml"""
use = """usage: ./run.py --loaddata --gencase --init
--schedule map_file
--genreport run_id
--init or -i for short, to clean up and init logger database
--schedule or -s for short, to load the schedule file and run
--loaddata or -l for short, to load data or not
--gencase or -g for short, to generate test cases
--genreport or -G for short, to generate test report by run_id
"""
parser = ArgumentParser(description=use)
parser.add_argument("-i",
"--initbenchmark",
action='store_true',
help="Initial benchmark db.")
parser.add_argument("-g",
"--gencase",
action='store_true',
help="Generate cases.")
parser.add_argument(
"-d",
"--debug",
action='store_true',
help=
"Debug model will generate all sql for each case, it take a long time."
)
parser.add_argument(
"-l",
"--forceload",
action='store_true',
help=
"Reconvert and reload all tables set in config.yaml and tables.yaml.")
parser.add_argument(
"-L",
"--smartload",
action='store_true',
help=
"Load modules by config.yaml, tables.yaml and -m. If table exists in db, do nothing."
)
parser.add_argument(
"-m",
"--module",
nargs="*",
help=
"Modules selected to load. For exapme: 'run.py -Lm Dec Ran' means only load decisition tree and random forest."
)
parser.add_argument("-s", "--schedule", help="Set schedule file and run.")
options = parser.parse_args()
testConfiger = test_config.Configer(Path.TestConfigXml)
testConfiger.testconfig()
analyticsTools = analytics_tool.AnalyticsTools(Path.AnalyticsToolXml)
analyticsTools.parseTools()
psql_args = testConfiger.getResultDBmanager().getDBsqlArgs()
schema = testConfiger.getResultSchema()
tools.set_search_path(schema, Path.BootstrapDir)
if options.schedule:
map_file = options.schedule
plans = tools.parserMap(Path.ScheduleDir + map_file)
run_sql.runSQL("update %s.testitemseq set runid = runid + 1;" % schema,
psqlArgs=psql_args)
ret = run_sql.runSQL("select max(runid) from %s.testitemseq;" % schema,
psqlArgs=psql_args)
run_id = ret.splitlines()[2].strip()
for plan in plans:
if len(plan) > 4:
sys.exit(
'ERROR:-s arg file has some grammer error, too many lines.'
)
if not 'skip' in plan:
plan['skip'] = ""
if 'cases' in plan:
isList = False
filename = plan['cases']
if 'lists' in plan:
isList = True
filename = plan['lists']
if 'unique' in plan and plan['unique']:
isUnique = True
else:
isUnique = False
version = run_case.runCases(filename, plan['skip'], isList,
isUnique, plan['platform'],
analyticsTools, testConfiger, run_id)
if plan['skip']:
run_sql.runSQL(Path.BootstrapDir + 'skipsqlfile.sql',
psqlArgs=psql_args,
isFile=True)
run_sql.runSQL(Path.BootstrapDir + 'post.sql',
psqlArgs=psql_args,
onErrorStop=False,
isFile=True)
gen_report.generate_report(psql_args, schema, run_id,
plan['platform'], Path.ReportDir)
#load data set to all databases to test
if options.forceload:
loading_manager = loadingManager(Path.RootPath, 'madlibtestdata',
analyticsTools)
loading_manager.do(options.module, False, True, True)
if options.initbenchmark:
print "------------ Initializing database ------------"
run_sql.runSQL(Path.BootstrapDir + 'init.sql',
psqlArgs=psql_args,
isFile=True)
run_sql.runSQL(Path.BootstrapDir + 'init_cases.sql',
psqlArgs=psql_args,
isFile=True)
run_sql.runSQL(Path.BootstrapDir + 'resultbaseline.sql',
psqlArgs=psql_args,
isFile=True)
if options.gencase:
print "------------ Generating new test cases ------------"
if options.debug:
os.system('cd ../src/generator/ && python ./gen_testcase.py debug')
else:
os.system('cd ../src/generator/ && python ./gen_testcase.py')
run_sql.runSQL(Path.BootstrapDir + 'analyticstool.sql',
psqlArgs=psql_args,
isFile=True)
print "------------ Initializing algorithm result table ------------"
run_sql.runSQL(Path.BootstrapDir + 'algorithmspec.sql',
psqlArgs=psql_args,
onErrorStop=False,
isFile=True)
for sqlfile in glob.glob('../testcase/*.sql'):
run_sql.runSQL(sqlfile,
psqlArgs=psql_args,
onErrorStop=False,
isFile=True)
if options.smartload:
loading_manager = loadingManager(Path.RootPath, 'madlibtestdata',
analyticsTools)
loading_manager.do(options.module, False, False, False)
