def find(basedir, matchstr, isDirectory=False, isRecursive=True): if isRecursive: matches = util.findMatchingFiles(basedir, matchstr, isDirectory) else: matches = util.findMatchingFiles(basedir, matchstr, isDirectory, 1) if matches: return matches[0] return ""
def getClasspathForJdbcClient(): jdbc_standalone_jar = getStandaloneHiveJdbcJar() assert jdbc_standalone_jar, "No JDBC standalone jar found" classpath = [jdbc_standalone_jar] hadoop_home = Config.get('hadoop', 'HADOOP_HOME') if Hadoop.isHadoop2() and Machine.type() == 'Windows': hadoop_home = os.path.join(hadoop_home, "share", "hadoop", "common") hadoop_common_jar = util.findMatchingFiles(hadoop_home, "hadoop-common-*[!(tests)].jar", depth=1) assert len(hadoop_common_jar) > 0, "No hadoop-common.jar found" classpath.append(hadoop_common_jar[0]) if Hadoop.isSecure(): hadoop_auth_jar = util.findMatchingFiles(hadoop_home, "hadoop-auth-*[!(tests)].jar", depth=1) assert len(hadoop_auth_jar) > 0, "No hadoop-auth.jar found" classpath.append(hadoop_auth_jar[0]) classpath.append(Config.get('hadoop', 'HADOOP_CONF')) return (os.pathsep).join(classpath)
def getVersion(cls): # determine the hive version from the jar that is deployed jarDir = os.path.join(cls._hcat_home, 'share', 'hcatalog') files = util.findMatchingFiles(jarDir, "*hcatalog-core-*.jar") p = re.compile('hcatalog-core-(\S+).jar') m = p.search(files[0]) if m: return m.group(1) else: return ""
def getVersion(cls): # determine the hive version from the jar that is deployed jarDir = os.path.join(cls.getHiveHome(), 'lib') files = util.findMatchingFiles(jarDir, "hive-metastore-*.jar") p = re.compile('hive-metastore-(\S+).jar') m = p.search(files[0]) if m: return m.group(1) else: return ""
def get_stack_trace(cls, test_name, current_dir): apiReportDirectory = os.path.join(current_dir, 'target', 'surefire-reports', 'junitreports') uifrmReportDirectory = os.path.join(current_dir, 'target', 'surefire-reports') pattern = '*' + UpgradeLogger.get_testclass_name(test_name) + '*.xml' testResultFiles = util.findMatchingFiles( apiReportDirectory, pattern ) # First search for test result xml in api framework dir if testResultFiles.__len__() == 0: testResultFiles = util.findMatchingFiles( uifrmReportDirectory, pattern ) # Now search for test result xml in uifrm dir Machine.runas( 'root', 'chmod -R 755 ' + uifrmReportDirectory ) # Provide read permissions to everyone in uifrm junit report xml directory else: Machine.runas( 'root', 'chmod -R 755 ' + apiReportDirectory ) # Provide read permissions to everyone in api framework report xml directory testresult = {} for resultFile in testResultFiles: testresult.update(util.parseJUnitXMLResult(resultFile)) testOutput = {} for key, value in testresult.items(): m = re.search("([^\.]*)$", key) # Extract test name print "key : %s " % key print "value : %s " % value key = m.group(0) fail_message = value['failure'] print "Final key: %s" % key print "Final fail_msg: %s" % fail_message if fail_message: testOutput[str(key)] = fail_message print testOutput return testOutput
def getdnjars(cls): # Workaround BUG-58287 org.datanucleus.api.jdo.JDOPersistenceManagerFactory spark_lib_dir = os.path.join(Spark.getSparkHome(), "lib") dn_jars = util.findMatchingFiles(spark_lib_dir, "datanucleus*.jar") jars = '' for jar in dn_jars: jars = jar + "," + jars # remove the last "," in the list jars = jars[:-1] return jars
def gatherTestResults(self): logger.info("=====Gathering results=========") testresult = {} global nonPyTestReportList Machine.runas( 'root', 'chmod -R 755 ' + os.path.join(self.LOCAL_WORK_DIR, 'target')) # get a list of all the test result files testResultFiles = util.findMatchingFiles( os.path.join(self.LOCAL_WORK_DIR, 'target', 'surefire-reports'), 'TEST-*.xml') #print "Test result files: " % testResultFiles for resultFile in testResultFiles: testresult.update(util.parseJUnitXMLResult(resultFile)) for key, value in testresult.items(): fail_message = value['failure'] print "key : %s " % key print "value : %s " % value print "fail_mesg : %s " % fail_message m = re.search("([^\.]*)$", key) #remove braces and hyphens from tc name #key = m.group(1).replace('-','') key = 'split-' + self.splitNumStr + '-' + m.group(1) #if '[' in key: # key = key[:key.index('[')] value = re.match(r'.*result(.*)', str(value)).group(1) #remove additional quotes from tc status if '\'' in value: value = value[3:9].replace('\'', "") self.TEST_RESULT[str(key)] = { "status": str(value), "failure": fail_message } self.TESTCASES.extend(sorted(self.TEST_RESULT.keys())) for testcase in self.TESTCASES: print(testcase) TEST_SUITE_REPORT = dict() jsonFile = os.path.join('/tmp', 'testTimeDurations.json') if os.path.exists(jsonFile): Machine.runas('root', 'chmod -R 777 /tmp') with open(jsonFile, 'r') as data_file: data = json.load(data_file) testcases = data['testCases'] for testcase in testcases: testcasename = 'split-' + self.splitNumStr + '-' + testcase[ 'testCaseName'] TEST_SUITE_REPORT[testcasename] = testcase nonPyTestReportList.append( {testcasename: TEST_SUITE_REPORT[testcasename]}) print nonPyTestReportList
def getVersion(cls): # determine the sqoop version from the jar that is deployed jarDir = os.path.join(Config.get('sqoop2', 'SQOOP2_HOME'), 'shell-lib') files = util.findMatchingFiles(jarDir, "sqoop-client-*.jar") p = re.compile('sqoop-client-(\S+).jar') if files: m = p.search(files[0]) if m: return m.group(1) else: return ""
def getVersion(cls): jarDir = Config.get("phoenix", "PHOENIX_HOME") files = util.findMatchingFiles(jarDir, "phoenix-*-thin-client.jar") jarFileName = files[0].split(os.path.sep)[-1] p = re.compile(r'phoenix-(\S+)-thin-client.jar') m = p.search(jarFileName) if m: return m.group(1) else: return ""
def getVersion(cls): ''' Returns Druid user ''' jarDir = os.path.join(cls.getDruidHome(), 'lib') logger.info(jarDir) files = util.findMatchingFiles(jarDir, "druid-server-*.jar") p = re.compile('druid-server-(\S+).jar') m = p.search(files[0]) if m: return m.group(1) else: return ""
def run_smoke_test(cls, smoketestnumber, config=None): ''' Run smoke test for yarn :param smoketestnumber: Used for unique output log location ''' global new_conf_path global buildNo from beaver.component.hbase import HBase # Run slider agent labels funtests from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Slider smoke test started ####") exit_code, stdout = HBase.runShellCmds(cls.SCAN_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path) UpgradePerNode.reportProgress("### Slider smoke test scanned " + cls.TABLE_NAME) hbase_ver = HBase.getVersion(configPath=new_conf_path) if buildNo not in hbase_ver: UpgradePerNode.reportProgress("### smoke test failed: " + buildNo + " not found in " + hbase_ver) else: UpgradePerNode.reportProgress("### smoke test passed: " + buildNo + " found in " + hbase_ver) UpgradePerNode.reportProgress("scanned " + cls.TABLE_NAME) TABLE_NM = cls.id_generator(10) CREATE_TABLE_CMD = ["create '" + TABLE_NM + "', 'family1', 'family2', 'family3'"] exit_code, stdout = HBase.runShellCmds(CREATE_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path) if exit_code == 0: UpgradePerNode.reportProgress("created " + TABLE_NM) else: UpgradePerNode.reportProgress("creation of " + TABLE_NM + "failed") if not cls._LOCAL_WORK_DIR: UpgradePerNode.reportProgress( "[FAILED][Slider][Smoke] Slider smoke test failed due to Slider source code unavailability" ) return exit_code, stdout = Maven.run( "verify -Dslider.conf.dir=%s " "-Dhadoop.version=%s " "-Dprivate.repo.url=%s " "-Dit.test=AppsUpgradeIT" % (cls._LOCAL_CONF_DIR, Hadoop.getVersion(), Maven.getPublicRepoUrl()), cwd=os.path.join(cls._LOCAL_WORK_DIR, 'slider-funtest') ) testresults = {} testResultFiles = [] TEST_RESULT = {} # get a list of all the test result files for name, dirs, files in os.walk(cls._LOCAL_WORK_DIR): if os.path.basename(name) == 'target': # Add in each failsafe-report we find -- this test only runs failsafe reports testResultFiles.extend(util.findMatchingFiles(os.path.join(name, 'failsafe-reports'), 'TEST-*.xml')) for resultFile in testResultFiles: testresults.update(util.parseJUnitXMLResult(resultFile)) for key, value in testresults.items(): TEST_RESULT[key] = value logger.info("=======================") logger.info(TEST_RESULT) logger.info("=======================") TestCases = TEST_RESULT.keys() for testcase in TestCases: result = TEST_RESULT[testcase]['result'] if result == "pass": UpgradePerNode.reportProgress("[PASSED][Slider][Smoke] Slider smoke test passed") else: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider smoke test failed")
def find(basedir, matchstr): matches = util.findMatchingFiles(basedir, matchstr) if len(matches) > 0: return matches[0] return ""
def getStandaloneHiveJdbcJar(): hive_lib_dir = os.path.join(Hive.getHiveHome(), 'jdbc') jdbc_standalone_jar = util.findMatchingFiles(hive_lib_dir, "hive-jdbc-*-standalone.jar", depth=1) return jdbc_standalone_jar[0] if len(jdbc_standalone_jar) > 0 else None