def smoke_test_setup(cls): ''' Setup required to run Smoke test ''' from beaver.component.slider import Slider from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode HADOOPQA_USER = Config.get('hadoop', 'HADOOPQA_USER') UpgradePerNode.reportProgress("### Starting set up for Slider smoke test ####") if Hadoop.isSecure(): keytabFile = Machine.getHeadlessUserKeytab(HADOOPQA_USER) kinitloc = Machine.which("kinit", "root") cmd = "%s -k -t %s %s" % (kinitloc, keytabFile, Machine.get_user_principal(HADOOPQA_USER)) exit_code, stdout = Machine.run(cmd) if exit_code != 0: UpgradePerNode.reportProgress("### smoke test setup for Slider failed due to kinit failed ####") # Local directory in artifacts that we'll run tests from # it is possible the download_source() will fail try: cls._LOCAL_WORK_DIR = Slider.download_source(useHDPBaseRepoFile=False, isRUcluster=True) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) except TypeError as err: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source download fail.") return if not cls._LOCAL_WORK_DIR: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source Code missing.") return # Local conf directory with modified conf for tests if not os.path.exists(cls._LOCAL_CONF_DIR): os.makedirs(cls._LOCAL_CONF_DIR) slider_conf = os.path.join(Slider.getSliderHome(), 'conf') logger.info("slider_conf = %s" % slider_conf) logger.info("os path exist slider_conf = %s" % os.path.exists(slider_conf)) if os.path.exists(slider_conf): Machine.copy( os.path.join(slider_conf, 'log4j.properties'), os.path.join(cls._LOCAL_CONF_DIR, 'log4j.properties') ) Machine.copy( os.path.join(slider_conf, 'slider-env.sh'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-env.sh') ) else: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Conf %s missing" % slider_conf) return if Hadoop.isSecure(): util.writePropertiesToConfigXMLFile( os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), { "slider.funtest.enabled": "true", "slider.test.agent.enabled": "true", "HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'), "slider.am.keytab.local.path": Machine.getHeadlessUserKeytab(HADOOPQA_USER), "slider.keytab.principal.name": Machine.get_user_principal(HADOOPQA_USER) } ) else: util.writePropertiesToConfigXMLFile( os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), { "slider.funtest.enabled": "true", "HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'), "slider.test.agent.enabled": "true" } ) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) # Check that the precondition is met and the source is available if not os.path.exists(cls._LOCAL_WORK_DIR) or not os.path.exists(os.path.join(cls._LOCAL_WORK_DIR, 'pom.xml')): logger.info('Slider source does not appear to exist at %s' % (cls._LOCAL_WORK_DIR)) UpgradePerNode.reportProgress( "### Slider source does not appear to exist at %s ####" % (cls._LOCAL_WORK_DIR) ) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) if cls._LOCAL_WORK_DIR == None: logger.info("ERROR: cls._LOCAL_WORK_DIR is None") # Install first so isolated modules can be tested exit_code, stdout = Maven.run( "clean install -DskipTests " "-Dhadoop.version=%s " "-Dprivate.repo.url=%s " % (Hadoop.getVersion(), Maven.getPublicRepoUrl()), cwd=cls._LOCAL_WORK_DIR ) if exit_code != 0: UpgradePerNode.reportProgress("### Error installing Slider source : %d: %s ####" % (exit_code, stdout)) else: UpgradePerNode.reportProgress("### Slider source install passed ####")
def run_smoke_test(cls, smoketestnumber, config=None): ''' Run smoke test for yarn :param smoketestnumber: Used for unique output log location ''' global new_conf_path global buildNo from beaver.component.hbase import HBase # Run slider agent labels funtests from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Slider smoke test started ####") exit_code, stdout = HBase.runShellCmds(cls.SCAN_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path) UpgradePerNode.reportProgress("### Slider smoke test scanned " + cls.TABLE_NAME) hbase_ver = HBase.getVersion(configPath=new_conf_path) if buildNo not in hbase_ver: UpgradePerNode.reportProgress("### smoke test failed: " + buildNo + " not found in " + hbase_ver) else: UpgradePerNode.reportProgress("### smoke test passed: " + buildNo + " found in " + hbase_ver) UpgradePerNode.reportProgress("scanned " + cls.TABLE_NAME) TABLE_NM = cls.id_generator(10) CREATE_TABLE_CMD = ["create '" + TABLE_NM + "', 'family1', 'family2', 'family3'"] exit_code, stdout = HBase.runShellCmds(CREATE_TABLE_CMD, user=cls.HBASE_USER, configPath=new_conf_path) if exit_code == 0: UpgradePerNode.reportProgress("created " + TABLE_NM) else: UpgradePerNode.reportProgress("creation of " + TABLE_NM + "failed") if not cls._LOCAL_WORK_DIR: UpgradePerNode.reportProgress( "[FAILED][Slider][Smoke] Slider smoke test failed due to Slider source code unavailability" ) return exit_code, stdout = Maven.run( "verify -Dslider.conf.dir=%s " "-Dhadoop.version=%s " "-Dprivate.repo.url=%s " "-Dit.test=AppsUpgradeIT" % (cls._LOCAL_CONF_DIR, Hadoop.getVersion(), Maven.getPublicRepoUrl()), cwd=os.path.join(cls._LOCAL_WORK_DIR, 'slider-funtest') ) testresults = {} testResultFiles = [] TEST_RESULT = {} # get a list of all the test result files for name, dirs, files in os.walk(cls._LOCAL_WORK_DIR): if os.path.basename(name) == 'target': # Add in each failsafe-report we find -- this test only runs failsafe reports testResultFiles.extend(util.findMatchingFiles(os.path.join(name, 'failsafe-reports'), 'TEST-*.xml')) for resultFile in testResultFiles: testresults.update(util.parseJUnitXMLResult(resultFile)) for key, value in testresults.items(): TEST_RESULT[key] = value logger.info("=======================") logger.info(TEST_RESULT) logger.info("=======================") TestCases = TEST_RESULT.keys() for testcase in TestCases: result = TEST_RESULT[testcase]['result'] if result == "pass": UpgradePerNode.reportProgress("[PASSED][Slider][Smoke] Slider smoke test passed") else: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider smoke test failed")
"the cow jumped over the moon,1", "the man went to the store and bought some candy,2", "four score and seven years ago,3", "how many apples can you eat,4", "to be or not to be the person,5" ] HDFS_TRIDENT_SEQ_EXPECTED_LINES = [ "the cow jumped over the moon", "the man went to the store and bought some candy", "four score and seven years ago", "how many apples can you eat", "to be or not to be the person" ] HDFS_USER = Config.get('hadoop', 'HDFS_USER') HADOOPQA_USER = Config.get('hadoop', 'HADOOPQA_USER') HADOOP_VERSION = Hadoop.getVersion() HDFS_TOPOLOGY_NAME = "HDFSTopology" HDFS_SEQ_TOPOLOGY_NAME = "HDFSSequenceTopology" HDFS_TRIDENT_TOPOLOGY_NAME = "HDFSTridentTopology" HDFS_TRIDENT_SEQ_TOPOLOGY_NAME = "HDFSTridentSeqTopology" ######JDBC config###### STORM_JDBC_TEST_USER = "******" STORM_JDBC_TEST_PASSWD = "storm_test_password" JAVA_JDBC_SRC_DIR = os.path.join(Config.getEnv('WORKSPACE'), 'tests', 'storm', 'storm-jdbc', 'java') LOCAL_JDBC_WORK_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), 'storm-jdbc-tests') TARGET_JDBC_STORM_JAR = os.path.join( Config.getEnv('ARTIFACTS_DIR'), 'storm-jdbc-tests', 'target',