def setup(cls, S3_AWS_ACCESS_KEY=None, S3_AWS_SECRET=None): Machine.installPackageWithPip(packages="awscli", hosts=Hadoop.getAllNodes(), logoutput=True) aws_home = "/root/.aws" if not os.path.exists(aws_home): Machine.makedirs(ADMIN_USER, GATEWAY_NODE, "/root/.aws", ADMIN_PWD) util.writeToFile("[default]\nregion = us-west-2\noutput=json", os.path.join(ARTIFACTS_DIR, "config")) if S3_AWS_ACCESS_KEY: cls._aws_access_key_id = S3_AWS_ACCESS_KEY else: cls._aws_access_key_id = Config.get('machine', 'S3_AWS_ACCESS_KEY') if S3_AWS_SECRET: cls._aws_secret_access_key = S3_AWS_SECRET else: cls._aws_secret_access_key = Config.get( 'machine', 'S3_AWS_SECRET') util.writeToFile( "[default]\naws_access_key_id = %s\naws_secret_access_key = %s" % (cls._aws_access_key_id, cls._aws_secret_access_key), os.path.join(ARTIFACTS_DIR, "credentials")) Machine.runas( ADMIN_USER, "chown %s '%s/config'" % (ADMIN_USER, ARTIFACTS_DIR), GATEWAY_NODE, ADMIN_PWD) Machine.runas( ADMIN_USER, "chown %s '%s/credentials'" % (ADMIN_USER, ARTIFACTS_DIR), GATEWAY_NODE, ADMIN_PWD) Machine.copy(os.path.join(ARTIFACTS_DIR, "config"), aws_home, ADMIN_USER, ADMIN_PWD) Machine.copy(os.path.join(ARTIFACTS_DIR, "credentials"), aws_home, ADMIN_USER, ADMIN_PWD)
def setup_storm_hdfs_topology(cls, useStandaloneCmd): storm_version = Storm.getVersion(useStandaloneCmd=True) try: file_obj = open(HDFS_CONFIG_FILE, 'w') if Hadoop.isSecure(): file_obj.write('hdfs.keytab.file: ' + Machine.getHeadlessUserKeytab( user=HADOOPQA_USER) + '\n') file_obj.write('hdfs.kerberos.principal: ' + Machine.get_user_principal(user=HADOOPQA_USER) + '\n') finally: file_obj.close() HDFS.createDirectory("/tmp/mySeqTopology", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/dest", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/dest2", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/foo", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/trident", HDFS_USER, "777", False) HDFS.createDirectory("/tmp/trident-seq", HDFS_USER, "777", False) Machine.copy(JAVA_HDFS_SRC_DIR, LOCAL_HDFS_WORK_DIR, user=None, passwd=None) if not Machine.isWindows(): (exit_code, _) = Maven.run('package', cwd=LOCAL_HDFS_WORK_DIR, env={ HADOOP_VERSION_MAVEN_PARAMETER: HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER: storm_version, HADOOP_CONF_MAVEN_PARAMETER: HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER: HDFS_FILE, HADOOP_CORE_MAVEN_PARAMETER: HADOOP_CONF, CORE_FILE_MAVEN_PARAMETER: CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER: Maven.getPublicRepoUrl() }) else: filepath = os.path.join(MOD_CONF_PATH, "core-site.xml") (exit_code, _) = Maven.run( 'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s' % (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER, storm_version, HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE, HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF, CORE_FILE_MAVEN_PARAMETER, CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()), cwd=LOCAL_HDFS_WORK_DIR) ruAssert("Storm", exit_code == 0, "[StormHDFSSetup] maven package command failed")
def startLLAPWithChaosMonkey(interval='300'): hive_changes = {'tez-site.xml': {'tez.am.task.max.failed.attempts': '0'}} Hive.modifyConfig(hive_changes, services=['hiveserver2']) AMBARI_AGENT_TMP_DIR = '/var/lib/ambari-agent/tmp' ARTIFACTS_DIR = Config.getEnv('ARTIFACTS_DIR') LLAP_START_USER = Config.get('hive', 'HIVE_USER') dirs = [ name for name in os.listdir(AMBARI_AGENT_TMP_DIR) if os.path.isdir(os.path.join(AMBARI_AGENT_TMP_DIR, name)) ] llap_dirs = [] for dir in dirs: if dir.startswith('llap-slider'): llap_dirs.append(dir) if len(llap_dirs) < 1: logger.info("Could not find llap dir under %s" % AMBARI_AGENT_TMP_DIR) Hive.startService(services=['hiveserver2']) else: llap_dir = llap_dirs[-1] resourceConfig = os.path.join(AMBARI_AGENT_TMP_DIR, llap_dir, 'resources.json') tmpResourceConfig = os.path.join(ARTIFACTS_DIR, 'resources.json') propertyMap = [(["components", "LLAP"], {"yarn.container.failure.threshold": "1000"})] util.writePropertiesToConfigJSONFileMulti(resourceConfig, tmpResourceConfig, propertyMap) Machine.copy(tmpResourceConfig, resourceConfig, user=Machine.getAdminUser(), passwd=Machine.getAdminPasswd()) appConfig = os.path.join(AMBARI_AGENT_TMP_DIR, llap_dir, 'appConfig.json') tmpAppConfig = os.path.join(ARTIFACTS_DIR, 'appConfig.json') propertyMap = [ ( ["global"], { "internal.chaos.monkey.probability.containerfailure": "10000", "internal.chaos.monkey.interval.seconds": interval, "internal.chaos.monkey.enabled": "True" } ) ] util.writePropertiesToConfigJSONFileMulti(appConfig, tmpAppConfig, propertyMap) Machine.copy(tmpAppConfig, appConfig, user=Machine.getAdminUser(), passwd=Machine.getAdminPasswd()) llapShellScript = os.path.join(AMBARI_AGENT_TMP_DIR, llap_dir, 'run.sh') exit_code, stdout = Machine.runas(LLAP_START_USER, llapShellScript) if exit_code != 0: logger.info("LLAP Shell Script failed to run successfully with %d" % exit_code) for i in range(10): time.sleep(30) logger.info("@%d: Check if LLAP cluster is successfully deployed" % i) exit_code, stdout = Machine.runas(LLAP_START_USER, 'slider status llap0') if exit_code == 0: break elif i == 9: logger.info("LLAP cluster failed to deploy")
def setup_capacity_scheduler(cls, components): """ Setup yarn capacity scheduler based on components. This API is not called during setup_module. :param components: list of components :type components: list of str :return: None """ if RuSetup._defaultQueue: components.append("default") logger.info("*** setup_capacity_scheduler ***") if RuSetup._skipQueue != None: logger.info("Components do not have a queue: " + str(RuSetup._skipQueue)) components = list(set(components) - RuSetup._skipQueue) logger.info("components = %s" % components) numComponents = len(components) percentPerQueue = 100.0 / numComponents percentPerQueueStr = "{0:0.2f}".format(percentPerQueue) xmlDict = {} rootQueues = ",".join(components) xmlDict["yarn.scheduler.capacity.root.queues"] = rootQueues for component in components: xmlDict["yarn.scheduler.capacity.root.%s.capacity" % component] = percentPerQueueStr xmlDict["yarn.scheduler.capacity.root.%s.user-limit-factor" % component] = 1 xmlDict["yarn.scheduler.capacity.root.%s.maximum-capacity" % component] = percentPerQueueStr xmlDict["yarn.scheduler.capacity.root.%s.state" % component] = "RUNNING" xmlDict["yarn.scheduler.capacity.root.%s.acl_submit_jobs" % component] = "*" xmlDict["yarn.scheduler.capacity.root.%s.acl_administer_jobs" % component] = "*" util.dumpTextString(xmlDict, "====== PLANNED QUEUES ======", "==================") master_capacity_file = os.path.join(Config.getEnv("WORKSPACE"), "tests", "rolling_upgrade", "yarn", "data", "capacity-scheduler.xml") modified_capacity_file = os.path.join(Config.getEnv("ARTIFACTS_DIR"), "capacity-scheduler.xml") Machine.copy(master_capacity_file, modified_capacity_file) util.writePropertiesToConfigXMLFile(modified_capacity_file, modified_capacity_file, xmlDict) #util.dumpText(modified_capacity_file, "====== capacity-scheduler.xml ======", "==================") if RuSetup._defaultQueue: components.remove("default") return modified_capacity_file
def setup_storm_hbase_topology(cls, useStandaloneCmd): from beaver.component.hbase import HBase storm_version = Storm.getVersion(useStandaloneCmd=True) Machine.copy(JAVA_HBASE_SRC_DIR, LOCAL_HBASE_WORK_DIR, user=None, passwd=None) if Machine.isWindows(): (_, _) = Maven.run( 'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s' % (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER, storm_version, HBASE_CONF_MAVEN_PARAMETER, HBASE_CONF, HBASE_FILE_MAVEN_PARAMETER, HBASE_FILE, HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE, CORE_FILE_MAVEN_PARAMETER, CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()), cwd=LOCAL_HBASE_WORK_DIR) else: (_, _) = Maven.run('package', cwd=LOCAL_HBASE_WORK_DIR, env={ HADOOP_VERSION_MAVEN_PARAMETER: HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER: storm_version, HBASE_CONF_MAVEN_PARAMETER: HBASE_CONF, HBASE_FILE_MAVEN_PARAMETER: HBASE_FILE, HADOOP_CONF_MAVEN_PARAMETER: HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER: HDFS_FILE, CORE_FILE_MAVEN_PARAMETER: CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER: Maven.getPublicRepoUrl() }) exit_code, stdout = HBase.createTable(HBASE_TABLE_NAME, "cf", True, None) ruAssert("Storm", exit_code == 0) grant_cmd = "grant '%s', 'RWCXA', '%s', 'cf'" % (Storm.getStormUser(), HBASE_TABLE_NAME) exit_code, stdout = HBase.runShellCmds([grant_cmd])
def write_hbase_site(cls, hbase_site): tmpHBaseConfFile = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "hbase-site.xml") fout = open(tmpHBaseConfFile, "w") fout.write(hbase_site) fout.close() propertyMap = {'hbase.tmp.dir': '/tmp/hbase-tmp'} util.writePropertiesToConfigXMLFile(tmpHBaseConfFile, tmpHBaseConfFile, propertyMap) hbase_conf = Config.get('hbase', 'HBASE_CONF_DIR') hbaseConfFile = os.path.join(hbase_conf, "hbase-site.xml") Machine.copy(tmpHBaseConfFile, hbaseConfFile, user=Machine.getAdminUser(), passwd=Machine.getAdminPasswd()) return tmpHBaseConfFile
def setup_storm_jdbc_topology(cls, useStandaloneCmd): from beaver.dbutil import MySQL MySQL.createUserWithAllPriveleges(STORM_JDBC_TEST_USER, STORM_JDBC_TEST_PASSWD) storm_version = Storm.getVersion(useStandaloneCmd=True) try: file_obj = open(HDFS_CONFIG_FILE, 'w') if Hadoop.isSecure(): file_obj.write('hdfs.keytab.file: ' + Machine.getHeadlessUserKeytab( user=HADOOPQA_USER) + '\n') file_obj.write('hdfs.kerberos.principal: ' + Machine.get_user_principal(user=HADOOPQA_USER) + '\n') finally: file_obj.close() Machine.copy(JAVA_JDBC_SRC_DIR, LOCAL_JDBC_WORK_DIR, user=None, passwd=None) post_fenton_opt = " -DpostFenton=true" if Storm.isAfterFenton() else "" package_cmd = 'package ' + post_fenton_opt (exit_code, _) = Maven.run(package_cmd, cwd=LOCAL_JDBC_WORK_DIR, env={ HADOOP_VERSION_MAVEN_PARAMETER: HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER: storm_version, HADOOP_CONF_MAVEN_PARAMETER: HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER: HDFS_FILE, CORE_FILE_MAVEN_PARAMETER: CORE_FILE, PUBLIC_REPO_MAVEN_PARAMETER: Maven.getPublicRepoUrl() }) ruAssert("Storm", exit_code == 0, "[StormJDBCSetup] maven package command failed")
def smoke_test_setup(cls): ''' Setup required to run Smoke test ''' from beaver.component.slider import Slider from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode HADOOPQA_USER = Config.get('hadoop', 'HADOOPQA_USER') UpgradePerNode.reportProgress("### Starting set up for Slider smoke test ####") if Hadoop.isSecure(): keytabFile = Machine.getHeadlessUserKeytab(HADOOPQA_USER) kinitloc = Machine.which("kinit", "root") cmd = "%s -k -t %s %s" % (kinitloc, keytabFile, Machine.get_user_principal(HADOOPQA_USER)) exit_code, stdout = Machine.run(cmd) if exit_code != 0: UpgradePerNode.reportProgress("### smoke test setup for Slider failed due to kinit failed ####") # Local directory in artifacts that we'll run tests from # it is possible the download_source() will fail try: cls._LOCAL_WORK_DIR = Slider.download_source(useHDPBaseRepoFile=False, isRUcluster=True) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) except TypeError as err: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source download fail.") return if not cls._LOCAL_WORK_DIR: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Source Code missing.") return # Local conf directory with modified conf for tests if not os.path.exists(cls._LOCAL_CONF_DIR): os.makedirs(cls._LOCAL_CONF_DIR) slider_conf = os.path.join(Slider.getSliderHome(), 'conf') logger.info("slider_conf = %s" % slider_conf) logger.info("os path exist slider_conf = %s" % os.path.exists(slider_conf)) if os.path.exists(slider_conf): Machine.copy( os.path.join(slider_conf, 'log4j.properties'), os.path.join(cls._LOCAL_CONF_DIR, 'log4j.properties') ) Machine.copy( os.path.join(slider_conf, 'slider-env.sh'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-env.sh') ) else: UpgradePerNode.reportProgress("[FAILED][Slider][Smoke] Slider Conf %s missing" % slider_conf) return if Hadoop.isSecure(): util.writePropertiesToConfigXMLFile( os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), { "slider.funtest.enabled": "true", "slider.test.agent.enabled": "true", "HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'), "slider.am.keytab.local.path": Machine.getHeadlessUserKeytab(HADOOPQA_USER), "slider.keytab.principal.name": Machine.get_user_principal(HADOOPQA_USER) } ) else: util.writePropertiesToConfigXMLFile( os.path.join(Slider.getSliderHome(), 'conf', 'slider-client.xml'), os.path.join(cls._LOCAL_CONF_DIR, 'slider-client.xml'), { "slider.funtest.enabled": "true", "HADOOP_CONF_DIR": Config.get('hadoop', 'HADOOP_CONF'), "slider.test.agent.enabled": "true" } ) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) # Check that the precondition is met and the source is available if not os.path.exists(cls._LOCAL_WORK_DIR) or not os.path.exists(os.path.join(cls._LOCAL_WORK_DIR, 'pom.xml')): logger.info('Slider source does not appear to exist at %s' % (cls._LOCAL_WORK_DIR)) UpgradePerNode.reportProgress( "### Slider source does not appear to exist at %s ####" % (cls._LOCAL_WORK_DIR) ) logger.info("Local work dir = %s" % cls._LOCAL_WORK_DIR) if cls._LOCAL_WORK_DIR == None: logger.info("ERROR: cls._LOCAL_WORK_DIR is None") # Install first so isolated modules can be tested exit_code, stdout = Maven.run( "clean install -DskipTests " "-Dhadoop.version=%s " "-Dprivate.repo.url=%s " % (Hadoop.getVersion(), Maven.getPublicRepoUrl()), cwd=cls._LOCAL_WORK_DIR ) if exit_code != 0: UpgradePerNode.reportProgress("### Error installing Slider source : %d: %s ####" % (exit_code, stdout)) else: UpgradePerNode.reportProgress("### Slider source install passed ####")
def setup_storm_hive_topology(cls, useStandaloneCmd): from beaver.component.hive import Hive storm_version = Storm.getVersion(useStandaloneCmd=True) hive_version = Hive.getVersion() HIVE_METASTORE_URI = Hive.getConfigValue( "hive.metastore.uris", defaultValue="thrift://localhost:9083") global HIVE_METASTORE_URI global HIVE_HOST global HIVE_PORT global HIVE_WAREHOUSE_DIR HIVE_WAREHOUSE_DIR = Hive.getConfigValue( "hive.metastore.warehouse.dir", defaultValue="/apps/hive/warehouse") HIVE_HOST = Hive.getHiveHost() HIVE_PORT = Hive.getMetastoreThriftPort() if Storm.isDalorBeyond(): JAVA_HIVE_SRC_DIR = os.path.join(Config.getEnv('WORKSPACE'), 'tests', 'rolling_upgrade', 'Storm', '2_3', 'storm-hive', 'java') else: JAVA_HIVE_SRC_DIR = os.path.join(Config.getEnv('WORKSPACE'), 'tests', 'rolling_upgrade', 'Storm', '2_2', 'storm-hive', 'java') # hive.txn.manager and hive.support.concurrency are set through ambari as per bug-40500 #logger.info("Restart Hive") #changes = {'hive-site.xml': {'hive.txn.manager': 'org.apache.hadoop.hive.ql.lockmgr.DbTxnManager', # 'hive.support.concurrency': 'true'}} #Hive.modifyConfig(changes, services=['metastore'], restartService=True) logger.info("Create test database in Hive") exit_code, stdout = Hive.runQuery( cls.get_set_queue_cmd(useStandaloneCmd) + " drop database if exists stormdb cascade; \ create database stormdb;") ruAssert("Storm", exit_code == 0, "[StormHiveSetup] Failed to create test database" + stdout) HDFS.chmod(runasUser=HDFS.getHDFSUser(), perm=777, directory=HIVE_WAREHOUSE_DIR + "/" + DATABASE_NAME + ".db") #copy tests/storm/storm-hive/java to artifacts/storm-hive-tests logger.info("JAVA_SRC_DIR " + JAVA_HIVE_SRC_DIR) logger.info("LOCAL_WORK_DIR " + LOCAL_HIVE_WORK_DIR) Machine.copy(JAVA_HIVE_SRC_DIR, LOCAL_HIVE_WORK_DIR, user=None, passwd=None) #mvn package if Machine.isWindows(): (_, _) = Maven.run( 'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s' % (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER, storm_version, HIVE_VERSION_MAVEN_PARAMETER, hive_version, PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl(), CORE_FILE_MAVEN_PARAMETER, CORE_FILE, HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF, HIVE_CORE_MAVEN_PARAMETER, HIVE_CORE_DIR, HIVE_FILE_MAVEN_PARAMETER, HIVE_FILE), cwd=LOCAL_HIVE_WORK_DIR) else: (_, _) = Maven.run('package', cwd=LOCAL_HIVE_WORK_DIR, env={ HADOOP_VERSION_MAVEN_PARAMETER: HADOOP_VERSION, STORM_VERSION_MAVEN_PARAMETER: storm_version, HIVE_VERSION_MAVEN_PARAMETER: hive_version, PUBLIC_REPO_MAVEN_PARAMETER: Maven.getPublicRepoUrl(), CORE_FILE_MAVEN_PARAMETER: CORE_FILE, HADOOP_CONF_MAVEN_PARAMETER: HADOOP_CONF, HDFS_FILE_MAVEN_PARAMETER: HDFS_FILE, HADOOP_CORE_MAVEN_PARAMETER: HADOOP_CONF, HIVE_CORE_MAVEN_PARAMETER: HIVE_CORE_DIR, HIVE_FILE_MAVEN_PARAMETER: HIVE_FILE }) create_table_q = "use %s; \ drop table if exists %s; \ create table %s (id int, name string, phone string, street string) \ partitioned by (city string, state string) \ clustered by (id) into %s buckets \ stored as orc \ tblproperties ('transactional'='true');" % ( DATABASE_NAME, HIVE_TABLE_NAME, HIVE_TABLE_NAME, "5") exit_code, stdout = Hive.runQuery( cls.get_set_queue_cmd(useStandaloneCmd) + create_table_q) ruAssert( "Storm", exit_code == 0, "[StormHiveSetup] Failed to create test table userdata_partitioned" ) HDFS.chmod(runasUser=HDFS.getHDFSUser(), perm=777, directory=HIVE_WAREHOUSE_DIR + "/" + DATABASE_NAME + ".db/" + HIVE_TABLE_NAME)