def __executeCaseFile(self, file_name, run_id, restart=False): """Execute case file with specific greenplum/postgres configuration. param file_name, file name of case param run_id, the index of this run param restart, dose the test need to restart greenplum """ connection_str = self.dbManager.getDBConnection() exe_append = " -C " + connection_str \ + " --run_id " + str(run_id) \ + " --analyticstool " + self.platform lines = open(file_name).readlines() for line in lines: line = line.strip() if line and line[0] != '#': if line.startswith('psql'): #this is a psql command, need to add connection options line = line.replace("psql", "", 1) run_sql.runSQL(line, psqlArgs=self.psql_append, source_path=self.cur_dbconf['env']) else: #this is a executor command, need to add run_id and connection options if restart: self.dbManager.stop() self.dbManager.start() subprocess.call(line + exe_append, shell=True)
def executeCase(self, test_case_path, run_id, restart=False): """Execute case files with specific greenplum/postgres configuration param test_case_path, path of all test case param run_id, the index of this run param restart, dose the test need to restart greenplum """ self.dbManager.start() run_sql.runSQL(self.init_sql, psqlArgs=self.psql_append, source_path=self.cur_dbconf['env']) self.__executeCaseWithTool(test_case_path, run_id, restart) version_sql = 'select madlib.version();' output = run_sql.runSQL(version_sql, psqlArgs=self.psql_append, source_path=self.cur_dbconf['env']) try: temp_list = output.split(',') self.version = temp_list[2] except Exception: self.version = "WRONG" self.dbManager.stop()
def __executeCaseFile(self, file_name, run_id, restart = False): """Execute case file with specific greenplum/postgres configuration. param file_name, file name of case param run_id, the index of this run param restart, dose the test need to restart greenplum """ connection_str = self.dbManager.getDBConnection() exe_append = " -C " + connection_str \ + " --run_id " + str(run_id) \ + " --analyticstool " + self.platform lines = open(file_name).readlines() for line in lines: line = line.strip() if line and line[0] != '#': if line.startswith('psql'): #this is a psql command, need to add connection options line = line.replace("psql", "", 1) run_sql.runSQL(line, psqlArgs = self.psql_append, source_path = self.cur_dbconf['env']) else: #this is a executor command, need to add run_id and connection options if restart: self.dbManager.stop() self.dbManager.start() subprocess.call(line + exe_append, shell = True)
def gen_failedcasesreport(self): FailedCasesreport_sql = "SELECT DISTINCT casename FROM %s.failedcases;"% self.schema FailedCasesreport_filename = self.ReportDir + self.platform + '_' + self.run_id + '_FailedCases.report' copyToFile(self.psql_args, FailedCasesreport_sql, os.path.join(self.CWD, FailedCasesreport_filename)) dirpath = self.ReportDir + self.platform + '_' + self.run_id + "Failedcases_report" + '/' os.system('rm -rf ' + dirpath) os.system('mkdir -p ' + dirpath) sql="select %s.gen_failedreport( '%s', '%s')"%(self.schema, dirpath, self.schema) try: run_sql.runSQL(sql, psqlArgs = self.psql_args) except Exception,exp: print exp print '\nError when generating failed cases reports'
def run(self): """Run the madlib invoking sql statement, and store the result to logger database.""" is_successful = False sql = self.sql # print executor command and invocation sql statement print "----------------------------------------------------" print " ".join(self.argv) print self.sql timer = run_sql.Timer() # the elapsed time is in second, but we will convert it to ms in database with timer: try: result = run_sql.runSQL( sql, self.logusername, self.logpassword, self.loghostname, self.logport, self.logdatabase, ["--expanded"], ) is_successful = True except Exception as e: result = str(e) print str(e) try: # strip empty line, and use only the tail of 1k char result = "\n".join([l for l in result.splitlines() if l.strip() != ""]) result = result[-1000:] self.logger.log_test_item(timer.elapsed, is_successful, "table", result, sql, "false") if is_successful: self.__store_db_result(result) except Exception as e: print "ERROR", str(e)
def run(self): """Run the madlib invoking sql statement, and store the result to logger database.""" is_successful = False sql = self.sql #print executor command and invocation sql statement print '----------------------------------------------------' print ' '.join(self.argv) print self.sql timer = run_sql.Timer() #the elapsed time is in second, but we will convert it to ms in database with timer: try: result = run_sql.runSQL(sql, self.logusername, self.logpassword, self.loghostname, self.logport, self.logdatabase, ['--expanded']) is_successful = True except Exception as e: result = str(e) print str(e) try: #strip empty line, and use only the tail of 1k char result = '\n'.join( [l for l in result.splitlines() if l.strip() != '']) result = result[-1000:] self.logger.log_test_item(timer.elapsed, is_successful, "table", \ result, sql, "false") if is_successful: self.__store_db_result(result) except Exception as e: print 'ERROR', str(e)
def main(): psql_args= ['-U', 'gpdbchina', '-h', 'localhost', '-p', '5466', '-d', 'benchreport'] ret = run_sql.runSQL("select max(runid) from benchmark.testitemseq;", psqlArgs = psql_args) run_id = ret.splitlines()[2].strip() platform = "TEST" schema = "benchmark" generate_report(psql_args, schema, run_id, platform)
def copyToFile(psql_args, sql, filename): """generate test result report param sql as select, but we need to create as temp table to store those data and copy to file """ out = run_sql.runSQL(sql, psqlArgs = psql_args) f = open(filename, 'w') f.write(out) f.close()
def executeCase(self, test_case_path, run_id, restart=False): """Execute case files with specific greenplum/postgres configuration param test_case_path, path of all test case param run_id, the index of this run param restart, dose the test need to restart greenplum """ self.dbManager.start() run_sql.runSQL(self.init_sql, psqlArgs=self.psql_append, source_path=self.cur_dbconf["env"]) self.__executeCaseWithTool(test_case_path, run_id, restart) version_sql = "select madlib.version();" output = run_sql.runSQL(version_sql, psqlArgs=self.psql_append, source_path=self.cur_dbconf["env"]) try: temp_list = output.split(",") self.version = temp_list[2] except Exception: self.version = "WRONG"
def __load(self, db_manager, kind, yaml_list, overload=False): """Load table into db.""" fail_list = [] for yaml_path in yaml_list: if os.path.exists(os.path.join(self.__yamlPath, yaml_path)) \ is False: continue yaml_content = yaml.load(open(os.path.join(self.__yamlPath, yaml_path))) if 'tables' in yaml_content: for table in yaml_content['tables']: if 'skip' in table and \ (table['skip'] == 'all' or (table['skip'] in kind)): continue table_name = '.'.join([self.__schema, table['id']]) outSQL = os.path.join(self.__yamlPath, 'sql', table['id'] + '.sql') output = run_sql.runSQL("SELECT count(*) FROM %s" % table_name, psqlArgs=db_manager.getDBsqlArgs(), onErrorStop=False, Return="all") #If table exists and no nedd to overload, skip this sql. if output.find('not exist') < 0 and \ output.find(' 0') < 0 and overload is False: continue elif output.find(' 0') > 0 and overload is False: fail_list.append(table['id']) print "ERROR : Success create but copy failed : %s" % table['id'] continue try: start = time.time() subprocess.check_call('gunzip -f %s.gz' % outSQL, shell=True) run_sql.runSQL(outSQL, logport=db_manager.db_conf['port'], logdatabase=db_manager.db_conf['database'], onErrorStop=False, isFile=True, source_path=db_manager.getDBenv()) subprocess.check_call('gzip -f %s' % outSQL, shell=True) self.__logInfo(table['id'], 'load', time.time() - start) #Load additional sql file for table. if 'sql' in table: run_sql.runSQL(os.path.join(self.__yamlPath, os.path.dirname(yaml_path), table['sql']), logport=db_manager.db_conf['port'], logdatabase=db_manager.db_conf['database'], onErrorStop=False, isFile=True, source_path=db_manager.getDBenv()) print "INFO : Success Loaded : %s " % table['id'] except: fail_list.append(table['id']) print "ERROR : Fail Loaded : %s " % table['id'] #Load additional sql file for algorithm. if 'sql' in yaml_content: run_sql.runSQL(os.path.join(self.__yamlPath, os.path.dirname(yaml_path), yaml_content['sql']), logport=db_manager.db_conf['port'], logdatabase=db_manager.db_conf['database'], onErrorStop=False, isFile=True, source_path=db_manager.getDBenv()) print "FAILED LOAD TABLES:\n", fail_list
def runCases(getfile, skipfile, isList, isUnique, platform, analyticsTool, testConfiger, run_id): """run the cases specialized by getfile. """ (cases, skippedcases) = __parserCasesFromFile(getfile, skipfile, isList, isUnique, testConfiger) if not platform in analyticsTool.analyticsTools: sys.exit('ERROR: Wrong platform name.') testdbconf = analyticsTool.analyticsTools[platform] executor = execute_case.TestCaseExecutor(cases, testdbconf, platform) executor.executeCase(Path.TestCaseDir, run_id) if skipfile: for case in skippedcases: f = open(Path.TestCaseDir + case + '.case') lines = f.readlines() for line in lines: if len(line) < 10: continue pos = line.find('target_base_name') if pos > 0: target_base_name = line[pos + 17:].strip() else: continue print target_base_name pos = line.find('-c') cmd = line[pos + 3:].strip() sql = """insert into %s.testitemresult values( '%s', %s, %s, '%s', '%s', %s, %s, '%s', '%s', %s::bool); """ % (testConfiger.getResultSchema(), target_base_name, run_id, \ 0, 'table', platform, 0, 'NULL', 'NULL', cmd, False) result = run_sql.runSQL(sql, testConfiger.user, None, testConfiger.host, testConfiger.port, testConfiger.database, ['--expanded']) return executor.version
def runCases(getfile, skipfile, isList, isUnique, platform, analyticsTool, testConfiger, run_id): """run the cases specialized by getfile. """ (cases, skippedcases) = __parserCasesFromFile(getfile, skipfile, isList, isUnique, testConfiger) if not platform in analyticsTool.analyticsTools: sys.exit('ERROR: Wrong platform name.') testdbconf = analyticsTool.analyticsTools[platform] executor = execute_case.TestCaseExecutor(cases, testdbconf, platform) executor.executeCase(Path.TestCaseDir, run_id) if skipfile: for case in skippedcases: f = open( Path.TestCaseDir +case+'.case') lines = f.readlines() for line in lines: if len(line) < 10: continue pos =line.find('target_base_name') if pos > 0: target_base_name = line[pos + 17:].strip() else: continue print target_base_name pos = line.find('-c') cmd = line[pos + 3:].strip() sql = """insert into %s.testitemresult values( '%s', %s, %s, '%s', '%s', %s, %s, '%s', '%s', %s::bool); """ % (testConfiger.getResultSchema(), target_base_name, run_id, \ 0, 'table', platform, 0, 'NULL', 'NULL', cmd, False) result = run_sql.runSQL(sql, testConfiger.user, None, testConfiger.host, testConfiger.port, testConfiger.database,['--expanded']) return executor.version
def __load(self, db_manager, kind, yaml_list, overload=False): """Load table into db.""" fail_list = [] for yaml_path in yaml_list: if os.path.exists(os.path.join(self.__yamlPath, yaml_path)) is False: continue yaml_content = yaml.load(open(os.path.join(self.__yamlPath, yaml_path))) if "tables" in yaml_content: for table in yaml_content["tables"]: if "skip" in table and (table["skip"] == "all" or (table["skip"] in kind)): continue table_name = ".".join([self.__schema, table["id"]]) outSQL = os.path.join(self.__yamlPath, "sql", table["id"] + ".sql") output = run_sql.runSQL( "SELECT count(*) FROM %s" % table_name, psqlArgs=db_manager.getDBsqlArgs(), onErrorStop=False, Return="all", ) # If table exists and no nedd to overload, skip this sql. if output.find("not exist") < 0 and output.find(" 0") < 0 and overload is False: continue elif output.find(" 0") > 0 and overload is False: fail_list.append(table["id"]) print "ERROR : Success create but copy failed : %s" % table["id"] continue try: start = time.time() subprocess.check_call("gunzip -f %s.gz" % outSQL, shell=True) run_sql.runSQL( outSQL, logport=db_manager.db_conf["port"], logdatabase=db_manager.db_conf["database"], onErrorStop=False, isFile=True, source_path=db_manager.getDBenv(), ) subprocess.check_call("gzip -f %s" % outSQL, shell=True) self.__logInfo(table["id"], "load", time.time() - start) # Load additional sql file for table. if "sql" in table: run_sql.runSQL( os.path.join(self.__yamlPath, os.path.dirname(yaml_path), table["sql"]), logport=db_manager.db_conf["port"], logdatabase=db_manager.db_conf["database"], onErrorStop=False, isFile=True, source_path=db_manager.getDBenv(), ) print "INFO : Success Loaded : %s " % table["id"] except: fail_list.append(table["id"]) print "ERROR : Fail Loaded : %s " % table["id"] # Load additional sql file for algorithm. if "sql" in yaml_content: run_sql.runSQL( os.path.join(self.__yamlPath, os.path.dirname(yaml_path), yaml_content["sql"]), logport=db_manager.db_conf["port"], logdatabase=db_manager.db_conf["database"], onErrorStop=False, isFile=True, source_path=db_manager.getDBenv(), ) print "FAILED LOAD TABLES:\n", fail_list
def main(): """Change the options, del -c -S -t -r. Modify -s. i.e. python run.py -s map.yaml""" use = """usage: ./run.py --loaddata --gencase --init --schedule map_file --genreport run_id --init or -i for short, to clean up and init logger database --schedule or -s for short, to load the schedule file and run --loaddata or -l for short, to load data or not --gencase or -g for short, to generate test cases --genreport or -G for short, to generate test report by run_id """ parser = ArgumentParser(description=use) parser.add_argument("-i", "--initbenchmark", action='store_true', help = "Initial benchmark db.") parser.add_argument("-g", "--gencase", action='store_true', help = "Generate cases.") parser.add_argument("-d", "--debug", action = 'store_true', help ="Debug model will generate all sql for each case, it take a long time.") parser.add_argument("-l", "--forceload", action='store_true', help = "Reconvert and reload all tables set in config.yaml and tables.yaml.") parser.add_argument("-L", "--smartload", action='store_true', help = "Load modules by config.yaml, tables.yaml and -m. If table exists in db, do nothing.") parser.add_argument("-m", "--module", nargs = "*", help = "Modules selected to load. For exapme: 'run.py -Lm Dec Ran' means only load decisition tree and random forest.") parser.add_argument("-s", "--schedule", help = "Set schedule file and run.") options = parser.parse_args() testConfiger = test_config.Configer(Path.TestConfigXml) testConfiger.testconfig() analyticsTools = analytics_tool.AnalyticsTools(Path.AnalyticsToolXml) analyticsTools.parseTools() psql_args = testConfiger.getResultDBmanager().getDBsqlArgs() schema = testConfiger.getResultSchema() tools.set_search_path(schema, Path.BootstrapDir) if options.schedule: map_file = options.schedule plans = tools.parserMap(Path.ScheduleDir + map_file) run_sql.runSQL("update %s.testitemseq set runid = runid + 1;"%schema, psqlArgs = psql_args) ret = run_sql.runSQL("select max(runid) from %s.testitemseq;"%schema, psqlArgs = psql_args) run_id = ret.splitlines()[2].strip() for plan in plans: if len(plan) > 4: sys.exit('ERROR:-s arg file has some grammer error, too many lines.') if not 'skip' in plan: plan['skip'] = "" if 'cases' in plan: isList = False filename = plan['cases'] if 'lists' in plan: isList = True filename = plan['lists'] if 'unique' in plan and plan['unique']: isUnique = True else: isUnique = False version = run_case.runCases(filename, plan['skip'], isList, isUnique, plan['platform'], analyticsTools, testConfiger, run_id) if plan['skip']: run_sql.runSQL(Path.BootstrapDir + 'skipsqlfile.sql', psqlArgs = psql_args, isFile = True) run_sql.runSQL(Path.BootstrapDir + 'post.sql', psqlArgs = psql_args, onErrorStop = False, isFile = True) gen_report.generate_report(psql_args, schema, run_id, plan['platform'], Path.ReportDir) #load data set to all databases to test if options.forceload: loading_manager = loadingManager(Path.RootPath, 'madlibtestdata', analyticsTools) loading_manager.do(options.module, False, True, True) if options.initbenchmark: #initialization run_sql.runSQL(Path.BootstrapDir + 'init.sql', psqlArgs = psql_args, isFile = True) run_sql.runSQL(Path.BootstrapDir + 'init_cases.sql', psqlArgs = psql_args, isFile = True) run_sql.runSQL(Path.BootstrapDir + 'resultbaseline.sql', psqlArgs = psql_args, isFile = True) #generate new cases os.system('cd ../src/generator/ && python ./gen_testcase.py') run_sql.runSQL(Path.BootstrapDir + 'analyticstool.sql', psqlArgs = psql_args, isFile = True) #initialize algorithm result table run_sql.runSQL(Path.BootstrapDir + 'algorithmspec.sql', psqlArgs = psql_args, isFile = True) for sqlfile in glob.glob('../testcase/*.sql'): run_sql.runSQL(sqlfile, psqlArgs = psql_args, onErrorStop = False, isFile = True) if options.gencase: #initialization run_sql.runSQL(Path.BootstrapDir + 'init_cases.sql', psqlArgs = psql_args, isFile = True) run_sql.runSQL(Path.BootstrapDir + 'resultbaseline.sql', psqlArgs = psql_args, isFile = True) #generate new cases if options.debug: os.system('cd ../src/generator/ && python ./gen_testcase.py debug') else: os.system('cd ../src/generator/ && python ./gen_testcase.py') run_sql.runSQL(Path.BootstrapDir + 'analyticstool.sql', psqlArgs = psql_args, isFile = True) run_sql.runSQL(Path.BootstrapDir + 'algorithmspec.sql', psqlArgs = psql_args, onErrorStop = False, isFile = True) for sqlfile in glob.glob('../testcase/*.sql'): run_sql.runSQL(sqlfile, psqlArgs = psql_args, onErrorStop = False, isFile = True) if options.smartload: loading_manager = loadingManager(Path.RootPath, 'madlibtestdata', analyticsTools) loading_manager.do(options.module, False, False, False)
def main(): """Change the options, del -c -S -t -r. Modify -s. i.e. python run.py -s map.yaml""" use = """usage: ./run.py --loaddata --gencase --init --schedule map_file --genreport run_id --init or -i for short, to clean up and init logger database --schedule or -s for short, to load the schedule file and run --loaddata or -l for short, to load data or not --gencase or -g for short, to generate test cases --genreport or -G for short, to generate test report by run_id """ parser = ArgumentParser(description=use) parser.add_argument("-i", "--initbenchmark", action='store_true', help="Initial benchmark db.") parser.add_argument("-g", "--gencase", action='store_true', help="Generate cases.") parser.add_argument( "-d", "--debug", action='store_true', help= "Debug model will generate all sql for each case, it take a long time." ) parser.add_argument( "-l", "--forceload", action='store_true', help= "Reconvert and reload all tables set in config.yaml and tables.yaml.") parser.add_argument( "-L", "--smartload", action='store_true', help= "Load modules by config.yaml, tables.yaml and -m. If table exists in db, do nothing." ) parser.add_argument( "-m", "--module", nargs="*", help= "Modules selected to load. For exapme: 'run.py -Lm Dec Ran' means only load decisition tree and random forest." ) parser.add_argument("-s", "--schedule", help="Set schedule file and run.") options = parser.parse_args() testConfiger = test_config.Configer(Path.TestConfigXml) testConfiger.testconfig() analyticsTools = analytics_tool.AnalyticsTools(Path.AnalyticsToolXml) analyticsTools.parseTools() psql_args = testConfiger.getResultDBmanager().getDBsqlArgs() schema = testConfiger.getResultSchema() tools.set_search_path(schema, Path.BootstrapDir) if options.schedule: map_file = options.schedule plans = tools.parserMap(Path.ScheduleDir + map_file) run_sql.runSQL("update %s.testitemseq set runid = runid + 1;" % schema, psqlArgs=psql_args) ret = run_sql.runSQL("select max(runid) from %s.testitemseq;" % schema, psqlArgs=psql_args) run_id = ret.splitlines()[2].strip() for plan in plans: if len(plan) > 4: sys.exit( 'ERROR:-s arg file has some grammer error, too many lines.' ) if not 'skip' in plan: plan['skip'] = "" if 'cases' in plan: isList = False filename = plan['cases'] if 'lists' in plan: isList = True filename = plan['lists'] if 'unique' in plan and plan['unique']: isUnique = True else: isUnique = False version = run_case.runCases(filename, plan['skip'], isList, isUnique, plan['platform'], analyticsTools, testConfiger, run_id) if plan['skip']: run_sql.runSQL(Path.BootstrapDir + 'skipsqlfile.sql', psqlArgs=psql_args, isFile=True) run_sql.runSQL(Path.BootstrapDir + 'post.sql', psqlArgs=psql_args, onErrorStop=False, isFile=True) gen_report.generate_report(psql_args, schema, run_id, plan['platform'], Path.ReportDir) #load data set to all databases to test if options.forceload: loading_manager = loadingManager(Path.RootPath, 'madlibtestdata', analyticsTools) loading_manager.do(options.module, False, True, True) if options.initbenchmark: print "------------ Initializing database ------------" run_sql.runSQL(Path.BootstrapDir + 'init.sql', psqlArgs=psql_args, isFile=True) run_sql.runSQL(Path.BootstrapDir + 'init_cases.sql', psqlArgs=psql_args, isFile=True) run_sql.runSQL(Path.BootstrapDir + 'resultbaseline.sql', psqlArgs=psql_args, isFile=True) if options.gencase: print "------------ Generating new test cases ------------" if options.debug: os.system('cd ../src/generator/ && python ./gen_testcase.py debug') else: os.system('cd ../src/generator/ && python ./gen_testcase.py') run_sql.runSQL(Path.BootstrapDir + 'analyticstool.sql', psqlArgs=psql_args, isFile=True) print "------------ Initializing algorithm result table ------------" run_sql.runSQL(Path.BootstrapDir + 'algorithmspec.sql', psqlArgs=psql_args, onErrorStop=False, isFile=True) for sqlfile in glob.glob('../testcase/*.sql'): run_sql.runSQL(sqlfile, psqlArgs=psql_args, onErrorStop=False, isFile=True) if options.smartload: loading_manager = loadingManager(Path.RootPath, 'madlibtestdata', analyticsTools) loading_manager.do(options.module, False, False, False)