def resetNode(cls, nodename, hostlist,action): HADOOP_HOME = Config.get('hadoop', 'HADOOP_HOME') HADOOP_BIN = os.path.join(HADOOP_HOME,"bin") nodetype = { 'namenode': 'master', 'datanode': 'slave', 'jobtracker': 'master', 'tasktracker': 'slave', }[nodename] MAPRED_USER = Config.get('hadoop', 'MAPRED_USER') HDFS_USER = Config.get('hadoop', 'HDFS_USER') user = { 'namenode': HDFS_USER, 'datanode': MAPRED_USER, 'jobtracker': HDFS_USER, 'tasktracker': MAPRED_USER, }[nodename] cmd = os.path.join(HADOOP_BIN,action + "-" + nodetype + ".cmd") pattern = "(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" for host in hostlist: if not re.search(pattern,host): host = host.split(':')[0] host = socket.gethostbyname(host.replace('\n','').strip()) host = re.findall(pattern, host)[0] logger.info( action + " " + nodename + " on host " + host) Machine.runinbackgroundAs(user, cmd, host,HADOOP_BIN)
def upgrade_master(cls, version, config=None): ''' Upgrades Master services: :param version: Version to be upgraded to :param config: Config location ''' from beaver.component.rollingupgrade.ruCommon import hdpSelect from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress("### Knox Upgrade started ####") #Stop knox gateway and apacheDS Knox.stopKnox() Knox.stopLdap() node = Config.get('knox', 'KNOX_HOST').split(',')[0] #Switch to the new version hdpSelect.changeVersion("knox-server", version, node) #Start apacheDs and knox gateway service Knox.startLdap() Knox.startKnox() time.sleep(10) UpgradePerNode.reportProgress("### Knox Upgrade Finished ####")
def deregisterHiveServer2(cls, version=None): if cls.isDynamicServiceDiscoveryOn(): hiveHome = cls.getHiveHome(service="hiveserver2") hiveUser = Config.get('hive', 'HIVE_USER') if not version: version = cls.getVersion() deregCmd = os.path.join(hiveHome, "bin", "hive") + " --service hiveserver2 --deregister %s" % version return Machine.runas(hiveUser, deregCmd)
def getClasspathForJdbcClient(): jdbc_standalone_jar = getStandaloneHiveJdbcJar() assert jdbc_standalone_jar, "No JDBC standalone jar found" classpath = [jdbc_standalone_jar] hadoop_home = Config.get('hadoop', 'HADOOP_HOME') if Hadoop.isHadoop2() and Machine.type() == 'Windows': hadoop_home = os.path.join(hadoop_home, "share", "hadoop", "common") hadoop_common_jar = util.findMatchingFiles(hadoop_home, "hadoop-common-*[!(tests)].jar", depth=1) assert len(hadoop_common_jar) > 0, "No hadoop-common.jar found" classpath.append(hadoop_common_jar[0]) if Hadoop.isSecure(): hadoop_auth_jar = util.findMatchingFiles(hadoop_home, "hadoop-auth-*[!(tests)].jar", depth=1) assert len(hadoop_auth_jar) > 0, "No hadoop-auth.jar found" classpath.append(hadoop_auth_jar[0]) classpath.append(Config.get('hadoop', 'HADOOP_CONF')) return (os.pathsep).join(classpath)
def queryserver(cls, user, host, action, config=None, homePath=None, binFolder='bin'): if Machine.type() == 'Windows': return Machine.service("queryserver", action, host=host) else: if homePath is None: homePath = Config.get('phoenix', 'PHOENIX_HOME') cmd = os.path.join(homePath, binFolder, 'queryserver.py') env = None if config: env = {} env['HBASE_CONF_DIR'] = config cmd = "%s %s" % (cmd, action) exit_code, stdout = Machine.runas(user, cmd, host=host, env=env) # The command exited abnormally, don't run any follow-on checks if exit_code != 0: logger.warn("Failed to execute queryserver command. %d, %s", exit_code, stdout) return exit_code, stdout # Check if PQS is accepting HTTP request before returning if action == 'start': logger.info('Verifying that PQS is running') if Phoenix.verify_pqs_running(host, QUERYSERVER_PORT): return exit_code, stdout # Failed to verify PQS is running, bail out. raise Exception('Failed to connect to PQS on %s' % host) return exit_code, stdout
def getJMXData(url, modelerType, metric, defaultValue=None, tries=5, wait_time=15, user=None): count = 0 my_data = 0 if not user: user = Config.get('hadoop', 'YARN_USER', 'yarn') while (my_data is None or my_data is 0 or my_data is '') and count < tries: JMX_DATA = util.getJSONContent(url, user=user) for data in JMX_DATA['beans']: # check for None if data is not None and data['modelerType'] is not None and data[ 'modelerType'] == modelerType: my_data = data[str(metric)] break count += 1 time.sleep(wait_time) if my_data is None or my_data is 0 or my_data is '': return defaultValue else: return my_data
def HDFS_getNNLogDir(cls, logoutput=True): try: nnHostList = cls.HDFS_getNamenodes() if nnHostList: for nnHost in nnHostList: lines = Machine.find(user=Machine.getAdminUser(), host=nnHost, filepath=Config.get( 'hadoop', 'HADOOP_LOG_DIR'), searchstr="hadoop*-namenode-*.log", passwd=Machine.getAdminPasswd()) lines = util.prune_output(lines, Machine.STRINGS_TO_IGNORE) if lines: nnLogFilePath = lines[0] if nnLogFilePath is None: continue else: return util.getDir(nnLogFilePath) return None except Exception: if logoutput: logger.error( "Exception occured during HDFS_getNNLogDir() call") logger.error(traceback.format_exc()) return None
def importDmp(cls, dmpfile, database, host=None): pg_cmd = "psql -U%s" % Config.get( 'machine', 'POSTGRES_ROOT_USER', default='postgres') if host: pg_cmd += " -h" + host pg_cmd += " -d%s -f %s" % (database, dmpfile) return Machine.run(pg_cmd)
def getDatanodeCount(cls): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -report") if exit_code == 0: m = re.match(".*Datanodes available: (\d+) \(", output, re.DOTALL) if m: return int(m.group(1)) return 0
def getDatabaseFlavor(cls): SQOOP2_HOME = Config.get("sqoop2", "SQOOP2_HOME") SQOOP2_LIB = os.path.join(SQOOP2_HOME, 'sqoop-server', 'lib') db = '' if (config.find('/usr/share/java', "mysql-connector-java*.jar")): db = "mysql" # special handling for postgres version if (config.find('/usr/share/java', "postgresql91-jdbc.jar")): db = 'postgres-9.1' elif (config.find(SQOOP_LIB, "postgresql93-jdbc.jar")): db = "postgres-9.3" elif (config.find('/usr/share/java', "postgresql*.jar")): db = "postgres-8" if (config.find(SQOOP2_LIB, "ojdbc*.jar")): db = "oracle" if (config.find(SQOOP2_LIB, "nzjdbc*.jar")): if db == '': db = "netezza" else: db += ", netezza" return db
def submit_storm_hive_topology(cls, tcId, className, args, useStandaloneCmd): if Hadoop.isSecure(): if Config.hasOption('machine', 'USER_REALM'): user_realm = Config.get('machine', 'USER_REALM', '') else: nnKerbPrincipal = HDFS.getNameNodePrincipal(defaultValue='') atloc = nnKerbPrincipal.find("@") if (atloc != -1): user_realm = nnKerbPrincipal[atloc:] if user_realm != None: args += " " + Machine.getHeadlessUserKeytab( Config.getEnv('USER')) + " " + Config.getEnv( 'USER') + '@' + user_realm exit_code, stdout = Storm.runStormHdfsTopology( TARGET_HIVE_STORM_JAR, className, args, None, logoutput=True, inBackground=False, useStandaloneCmd=useStandaloneCmd) logger.info(exit_code) ruAssert("Storm", exit_code == 0, "[StormHiveSubmit] %s Failed" % (tcId))
def runas(cls, user, cmd, cwd=None, env=None, logoutput=True, runInBackground=False): runCmd = Config.get('pig', 'PIG_CMD') + " " + cmd # initialize env if not env: env = {} # get kerberos ticket if Hadoop.isSecure(): if user is None: user = Config.getEnv('USER') kerbTicket = Machine.getKerberosTicket(user) env['KRB5CCNAME'] = kerbTicket user = None if runInBackground: return Machine.runinbackgroundAs(user, runCmd, cwd=cwd, env=env) else: return Machine.runas(user, runCmd, cwd=cwd, env=env, logoutput=logoutput)
def run_background_job(cls, runSmokeTestSetup=True, config=None, flagFile="/tmp/flagFile"): ''' Uploads Files to HDFS before upgrade starts and runs long running sleep job in background :return: number of application started ''' # start long running application which performs I/O operations (BUG-23838) #from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode #UpgradePerNode.reportProgress("### Background application for HDFS started ####") #jobArgs = {"mapred.job.queue.name" : cls._queue} #HadoopJobHelper.runSleepJob(numOfMaps = 1, numOfReduce = 1, mapSleepTime = "10000000", reduceSleepTime = "100", extraJobArg = jobArgs, runInBackground = True, config = config, directoutput = False ) #MAPRED.triggerSleepJob("1", "0", "100000", "1000000", 1, background = True) # load generator HADOOP_TEST_JAR = cls.get_hadoop_test_jar() TEST_USER = Config.get('hadoop', 'HADOOPQA_USER') HDFS.deleteDirectory(flagFile) slavelist = HDFS.getDatanodes() jobCmd = 'jar %s NNloadGenerator -Dmapred.job.queue.name=%s -mr 3 %s -root %s -numOfThreads 5 -maxDelayBetweenOps 1000 -elapsedTime 36000 -flagFile %s' % ( HADOOP_TEST_JAR, cls._queue, cls._lgTestOutputDir, cls._lgTestDataDir, flagFile) Hadoop.runInBackground(jobCmd) time.sleep(15) return 1
def ru_rollback_state(cls): ''' Saved state is rolled back - upgrade is abandonded NOTE: this command will not return until namenode shuts down :return: ''' logger.info("[INFO][HDFS][Upgrade] HA Journal Node Upgrade Started ") hdfs_user = Config.get('hadoop', 'HDFS_USER') nodes = [] nodes.append(HDFS.getNamenodeByState('standby')) nodes.append(HDFS.getNamenodeByState('active')) logger.info("[INFO][HDFS][Upgrade] HA Namenode Upgrade Started") for node in nodes: HDFS.resetZkfc('stop', hosts=node.split()) HDFS.resetNamenode('stop', host=node) HDFS.resetNamenode('start', config=None, host=nodes[0], option=" -rollingUpgrade rollback") HDFS.resetZkfc('start', hosts=nodes[0].split()) # lets make sure the NN is out of safemode before we proceed to the next namenode HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[0]) command = "sudo su - -c 'hadoop namenode -bootstrapStandby -force' hdfs" (exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command, nodes[1], None, None, "True", Machine.getAdminPasswd()) ruAssert("HDFS", exitcode == 0, "hadoop namenode -bootstrapStandby -force") HDFS.resetNamenode('start', config=None, host=nodes[1], option="") HDFS.resetZkfc('start', hosts=nodes[1].split()) # lets make sure the NN is out of safemode before we proceed to the next namenode HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[1])
def getZipFile(cls, version=HBase.getVersionFromBuild(), isRU=False): # download for linux, no download for windows HBASE_VER_BUILD = version if Machine.isWindows(): zipFile = os.path.join( Config.get('slider', 'SLIDER_HOME'), "app-packages", "slider-hbase-app-win-package-%s.zip" % HBASE_VER_BUILD) return zipFile pkg_list = "pkg-list_qe.txt" path = os.path.join(Config.getEnv('ARTIFACTS_DIR'), pkg_list) #pkgUrl = Config.get('slider','APP_PKG_LIST') pkgUrl = Slider.getAppPackageBaseUrl( isRU) + "/slider-app-packages/" + pkg_list util.downloadUrl(pkgUrl, path) with open(path, 'r') as f: for line in f: if line.startswith("hbase_pkg_url="): url = line.strip()[14:] break zipFile = os.path.join( os.getcwd(), "slider-hbase-app-package-%s.zip" % HBASE_VER_BUILD) logger.info("downloading " + url) util.downloadUrl(url, zipFile) return zipFile
def get_hadoop_test_jar(cls): return Machine.find( user=Machine.getAdminUser(), host='', filepath=Config.get('hadoop', 'MAPRED_HOME'), searchstr='hadoop-mapreduce-client-jobclient-*-tests.jar', passwd=Machine.getAdminPasswd())[0]
def upgrade_slave(cls, version, node, config=None, logText="Upgrade"): ''' Upgrade HDFS slave sevice: Datanode :param version: latestVersion. :param node: The node name where DN is running ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][%s] Datanode %s for node %s to version %s started " % (logText, logText, node, version)) ipcPort = HDFS.getDataNodeIPCPort() cmd = "dfsadmin -shutdownDatanode %s:%s upgrade" % (node, ipcPort) exitcode, stdout = HDFS.runas(Config.get('hadoop', 'HDFS_USER'), cmd, env=None, logoutput=True, config=None, host=None, skipAuth=False) if exitcode != 0: UpgradePerNode.reportProgress( "[INFO][HDFS][%s] Datanode shutdownDatanode command failed for %s " % (logText, node)) HDFS.waitForDNDown(node, ipcPort, "ipc.client.connect.max.retries=1") from beaver.component.rollingupgrade.ruCommon import hdpSelect hdpSelect.changeVersion("hadoop-hdfs-datanode", version, node) HDFS.startDatanodes(config=config, nodes=[node]) cls.wait4DNLive(node) UpgradePerNode.reportProgress( "[INFO][HDFS][%s] Datanode %s for node %s to version %s finished " % (logText, logText, node, version))
def ru_prepare_save_state_for_upgrade(cls): ''' Prepare Namenode to save State for Upgrade ''' from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode UpgradePerNode.reportProgress( "[INFO][HDFS][Prepare] Preparing state for HDFS upgrade") # BUG-26726: we need to be in safemode only in non HA cluster if not HDFS.isHAEnabled(): exit_code, output = HDFS.runasAdmin("dfsadmin -safemode enter") ruAssert("HDFS", exit_code == 0, '[Preparation] enter safemode failed') exit_code, output = HDFS.runas( Config.get('hadoop', 'HDFS_USER'), "dfsadmin -Ddfs.client.test.drop.namenode.response.number=0 -rollingUpgrade prepare" ) ruAssert("HDFS", exit_code == 0, '[Preparation] -rollingUpgrade prepare failed') if not HDFS.isHAEnabled(): exit_code, output = HDFS.runasAdmin("dfsadmin -safemode leave") ruAssert("HDFS", exit_code == 0, '[Preparation] leave safemode failed') UpgradePerNode.reportProgress( "[INFO][HDFS][Prepare] Preparing state for HDFS upgrade finished ")
def backupDatabase(cls, database, backupfile, host=None): pg_cmd = "pg_dump -U%s" % Config.get( 'machine', 'POSTGRES_ROOT_USER', default='postgres') if host: pg_cmd += " -h" + host pg_cmd += " %s > %s" % (database, backupfile) return Machine.run(pg_cmd)
def getTTHostForAttemptId(cls,attemptID): HADOOP_JOBTRACKER_LOG = Config.get('hadoop', 'HADOOP_JOBTRACKER_LOG') f = open(HADOOP_JOBTRACKER_LOG,"r") for line in f: searchFor = re.search(".*" + attemptID + ".*tracker_.*/(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}).*",line) if searchFor != None: return searchFor.group(1)
def runSource(cls, file_, host, database, cwd=None): return Machine.run( "psql -h%s -U%s -d%s -f%s" % (host, Config.get('machine', 'POSTGRES_ROOT_USER', default='postgres'), database, file_), cwd=cwd)
def log_test_properties(cls, propChanges): splitNumStr = str(propChanges['SPLIT_NUM']) ambariTestClass = str(propChanges['AMBARI_TESTSNAMES']) print "splitNumStr is: ", splitNumStr print "Ambari tests to run: ", ambariTestClass print "Additional properties for run :", Config.get('ambari', 'ADDITIONAL_AMBARI_PROPS')
def smoke_test_setup(cls): ''' Setup function for HDFS smoke test ''' if not cls._SmokeInputDir: cls._SmokeInputDir = cls._base_hdfs_dir + "/smokeHdfsInput" HDFS.deleteDirectory(cls._SmokeInputDir, Config.get('hadoop', 'HADOOPQA_USER')) jobCmd = 'jar %s randomtextwriter \"-D%s=%s\" \"-D%s=%s\" %s' % ( Config.get('hadoop', 'HADOOP_EXAMPLES_JAR'), "mapreduce.randomtextwriter.totalbytes", "4096", "mapred.job.queue.name", cls._queue, cls._SmokeInputDir) exit_code, stdout = Hadoop.run(jobCmd) ruAssert( "HDFS", exit_code == 0, '[SmokeSetup] Randomtextwriter job failed and could not create data on hdfs' )
def Knox_getLogDir(cls, logoutput=True): try: return Config.get('knox', 'KNOX_LOG', 'knoxlogdirnotfound') except Exception: if logoutput: logger.error("Exception occured during Knox_getLogDir() call") logger.error(traceback.format_exc()) return None
def getOozieLogDir(cls, logoutput=False): ''' Returns oozie log dir. ''' returnValue = Config.get("oozie", "OOZIE_LOG_DIR") if logoutput: logger.info("Oozie.getOozieLogDir returns %s" % returnValue) return returnValue
def print_environment_details(self): UpgradeLogger.reportProgress("=====Environment Details=====\n ", True) is_hdp = self.STACK_TYPE.lower() == "hdp" UpgradeLogger.reportProgress( "Ambari URL : " + Ambari.getWebUrl(is_hdp=is_hdp), True) UpgradeLogger.reportProgress("Ambari OS : " + Ambari.getOS(), True) UpgradeLogger.reportProgress( "Stack Type : " + Config.get('ambari', 'STACK_TYPE'), True) UpgradeLogger.reportProgress( "Ambari DB : " + Config.get('ambari', 'AMBARI_DB'), True) UpgradeLogger.reportProgress( "Kerberos : " + Config.get('machine', 'IS_SECURE'), True) UpgradeLogger.reportProgress( "HA : " + Config.get('ambari', 'IS_HA_TEST'), True) UpgradeLogger.reportProgress( "Wire Encryption : " + Config.get('ambari', 'WIRE_ENCRYPTION'), True)
def runAvroClient(cls, host, port, conffile): flume_conf = Config.get('flume-ng', 'FLUME_CONF') cmd = "avro-client -H %s -p %s" % (host, port) if Machine.type() == 'Linux': cmd += " -c %s -F %s" % (flume_conf, conffile) else: cmd += " -conf %s -filename %s" % (flume_conf, conffile) return cls.run(cmd)
def upgradeSchema(cls): hiveHome = cls.getHiveHome(service="metastore") hiveUser = Config.get('hive', 'HIVE_USER') hiveHost = cls.getHiveHost() dbflavor = cls.getDatabaseFlavor() if dbflavor.startswith("postgres"): dbflavor = "postgres" upCmd = os.path.join(hiveHome, "bin", "hive") + " --service schemaTool -dbType %s -upgradeSchema" % dbflavor return Machine.runas(hiveUser, upCmd, host=hiveHost)
def setupMergeScaleDataset(LOCAL_DIR): # change timezone on test machines Machine.resetTimeZoneOnCluster() # Download the TPCH dataset if not there tpch_data_dir = os.path.join(LOCAL_DIR, "data") TPCH_DATA_TGZ = os.path.join(LOCAL_DIR, "tpch_data.tgz") if not os.path.isfile(TPCH_DATA_TGZ): assert util.downloadUrl(Config.get('hive', 'TPCH_DNLD_URL'), TPCH_DATA_TGZ) Machine.tarExtractAll(TPCH_DATA_TGZ, LOCAL_DIR) # Load the tables in Hive HADOOPQA_USER = Config.get("hadoop", 'HADOOPQA_USER') HDFS.createDirectory("/tmp/tpch", user=HADOOPQA_USER, perm='777', force=True) HDFS.copyFromLocal(tpch_data_dir, "/tmp/tpch", user=HADOOPQA_USER) HDFS.chmod(None, 777, "/tmp/tpch", recursive=True) exit_code, stdout, stderr = Hive.runQueryOnBeeline( os.path.join(LOCAL_DIR, "ddl", "merge-tpch-tablesetup.sql"), hivevar={'HDFS_LOCATION': '/tmp/tpch/data'}, logoutput=True, queryIsFile=True ) assert exit_code == 0, "Failed to populate the TPCH data in Hive" # Download TPCH staging data tpch_stage_dir = os.path.join(LOCAL_DIR, "tpch_newdata_5G") TPCH_STAGE_TGZ = os.path.join(LOCAL_DIR, "tpch_newdata_5G.tgz") if not os.path.isfile(TPCH_STAGE_TGZ): assert util.downloadUrl(Config.get('hive', 'TPCH_NEWDATA_5G_DNLD_URL'), TPCH_STAGE_TGZ) Machine.tarExtractAll(TPCH_STAGE_TGZ, LOCAL_DIR) # Load the staged tables in Hive HDFS.createDirectory( "/tmp/lineitem_stage /tmp/orders_stage /tmp/delete_stage", user=HADOOPQA_USER, perm='777', force=True ) HDFS.copyFromLocal(os.path.join(tpch_stage_dir, "lineitem*"), "/tmp/lineitem_stage", HADOOPQA_USER) HDFS.copyFromLocal(os.path.join(tpch_stage_dir, "order*"), "/tmp/orders_stage", HADOOPQA_USER) HDFS.copyFromLocal(os.path.join(tpch_stage_dir, "delete*"), "/tmp/delete_stage", HADOOPQA_USER) HDFS.chmod(None, 777, "/tmp/lineitem_stage /tmp/orders_stage /tmp/delete_stage", recursive=True) exit_code, stdout, stderr = Hive.runQueryOnBeeline( os.path.join(LOCAL_DIR, "ddl", "merge-staged-tpch-tablesetup.sql"), hivevar={'HDFS_LOCATION': '/tmp'}, logoutput=True, queryIsFile=True ) assert exit_code == 0, "Failed to populate the TPCH staging data in Hive"
def getTempletonLogDir(cls, logoutput=False): ''' Returns Templeton log directory (String). ''' returnValue = Config.get('templeton', 'TEMPLETON_LOG_DIR', default='') if logoutput: logger.info("Hcatalog.getTempletonLogDir returns %s" % returnValue) return returnValue
def getLLAPDaemonPidsHosts(): hosts = [] llapdaemon_pids = [] if not Machine.isHumboldt(): nodes = Hadoop.getAllNodes() else: nodes = HDFS.getDatanodes() for node in nodes: pids = Machine.getProcessListRemote( node, format="%U %p %P %a", filter="org.apache.hadoop.hive.llap.daemon.impl.LlapDaemon", logoutput=True ) if pids: if Hadoop.isSecure(): pid = Machine.getPidFromString(pids[0], Config.get('hive', 'HIVE_USER')) else: pid = Machine.getPidFromString(pids[0], Config.get('hadoop', 'YARN_USER')) llapdaemon_pids.append(pid) hosts.append(node) return llapdaemon_pids, hosts
def runAsRoot(cls, cmd, host="", database=""): passwd = Config.get('machine', 'ORACLE_ROOT_PASSWD', default='root').strip() return cls.runJdbcCmd(cmd, user="******", passwd=passwd, host=host, database=database)
def runCmd(cls, cmd, host=None, database=None, user=None, passwd=None): if not user: user = Config.get('machine', 'POSTGRES_ROOT_USER', default='postgres') psql_cmd = Machine.echocmd(cmd) + " | psql -U%s" % user if host: psql_cmd += " -h" + host if database: psql_cmd += " -d" + database if not passwd: passwd = Config.get('machine', 'POSTGRES_ROOT_PASSWD', default='').strip() env = {} if passwd != "": env['PGPASSWORD'] = passwd return Machine.run(psql_cmd, env=env)
def triggerSleepJob(cls,numOfMaps,numOfReduce,mapsleeptime,reducesleeptime,numOfJobs,queue='',background=False): jobCounter = 0 while (jobCounter < numOfJobs): sleepCmd = " jar " + Config.get('hadoop', 'HADOOP_EXAMPLES_JAR') + " sleep " + queue + " -m " + numOfMaps + " -r " + numOfReduce + " -mt " + mapsleeptime +" -rt " + reducesleeptime if background: Hadoop.runInBackground(sleepCmd) else: Hadoop.run(sleepCmd) jobCounter = jobCounter + 1
def generateTestReportConf(infile, outfile, results): config = ConfigParser() config.optionxform=str config.read(infile) if config.has_section(SECTION): for option, value in config.items(SECTION): if value != "": continue elif option == "BUILD_ID" and config.has_option(SECTION, "REPO_URL"): config.set(SECTION, option, getBuildId(config.get(SECTION, "REPO_URL"))) config.remove_option(SECTION, "REPO_URL") elif option == "HOSTNAME": config.set(SECTION, option, socket.getfqdn()) elif option == "COMPONENT_VERSION": if not config.has_option(SECTION, "COMPONENT") or config.get(SECTION, "COMPONENT") == "": config.set(SECTION, "COMPONENT", "Hadoop") config.set(SECTION, option, getComponentVersion(config.get(SECTION, "COMPONENT"))) elif option == "OS": config.set(SECTION, option, platform.platform()) elif option == "SECURE" and Config.hasOption('hadoop', 'IS_SECURE'): config.set(SECTION, option, Config.get('hadoop', 'IS_SECURE').lower()) elif option == "BLOB": pass elif option == "RAN": config.set(SECTION, option, results[0] + len(results[1])) elif option == "PASS": config.set(SECTION, option, results[0]) elif option == "FAIL": config.set(SECTION, option, len(results[1])) elif option == "SKIPPED": config.set(SECTION, option, results[2]) elif option == "ABORTED": config.set(SECTION, option, results[3]) elif option == "FAILED_TESTS": config.set(SECTION, option, ",".join(results[1])) elif option == "SINGLE_NODE": from beaver.component.hadoop import HDFS if HDFS.getDatanodeCount() > 1: config.set(SECTION, option, "false") else: config.set(SECTION, option, "true") config.write(open(outfile, 'w'))
def runAdminReport(cls): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -report") return output
def refreshDatanodes(cls): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -refreshNodes") return output
def exitSafemode(cls): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -safemode leave") if re.search(".*Safe mode is OFF",output) != None: return True else: return False
def waitForNNOutOfSafemode(cls): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -safemode get") while (re.search(".*Safe mode is ON",output) != None): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "dfsadmin -safemode get") time.sleep(20) return True
#Get user from config file HADOOPQA_USER = CommonHadoopEnv.getHadoopQAUser() HDFS_USER = CommonHadoopEnv.getHDFSUser() MAPRED_USER = CommonHadoopEnv.getMapredUser() SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) DATA_PATH = os.path.join(SCRIPT_PATH, "data") CREATE_FILE = "CreateFile.py" CREATE_FILE_PATH_IN_HADOOP = '/user/' + HADOOPQA_USER + '/' + CREATE_FILE CREATE_FILE_PATH_IN_LOCAL = os.path.join(SCRIPT_PATH, "data", CREATE_FILE) CREATE_FILE_2 = "CreateFile2.py" CREATE_FILE_2_PATH_IN_HADOOP = '/user/' + HADOOPQA_USER + '/' + CREATE_FILE_2 CREATE_FILE_2_PATH_IN_LOCAL = os.path.join(SCRIPT_PATH, "data", CREATE_FILE_2) OUT_PATH_IN_HADOOP = '/user/' + HADOOPQA_USER + '/out1' HADOOP_STREAMING_JAR = Config.get('hadoop', 'HADOOP_STREAMING_JAR') logger = logging.getLogger(__name__) def validateJobId(jobId): jobId = jobId + "" return jobId.startswith('job_') def getLocalDirInfo(host): return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "mapred-site.xml"), "mapred.local.dir") def checkJobCreatedTempFileInTT(logFileDir, currentUser, currentJobId, currentAttemptId, logfile, taskTrackerHost): pathFile = os.path.join(logFileDir, 'taskTracker', currentUser, 'jobcache', currentJobId, currentAttemptId, 'work', logfile) logger.info("path file: " + pathFile) result = False if platform.system() == 'Windows':
def getHadoopExamplesJar(cls): return Config.get('hadoop', 'HADOOP_EXAMPLES_JAR')
def refreshTasktrackers(cls): exit_code, output = Hadoop.runas(Config.get('hadoop', 'HDFS_USER'), "mradmin -refreshNodes") return output
def getTasktrackers(cls): slaveFile = os.path.join(Config.get('hadoop', 'HADOOP_CONF'),"slaves") f = open(slaveFile, "r") tasktrackers = f.readlines() return tasktrackers
def run(cls, cmd, logoutput=True): return Machine.run(Config.get('hbase', 'HBASE_CMD') + " " + cmd, logoutput=logoutput)
def runInBackgroundAs(cls, user, cmd): hadoop_cmd = Config.get('hadoop', 'HADOOP_CMD') if Config.get('hadoop', 'HADOOP_CONF_EXCLUDE') == 'False': hadoop_cmd += " --config " + Config.get('hadoop', 'HADOOP_CONF') hadoop_cmd += " " + cmd return Machine.runinbackgroundAs(user, hadoop_cmd)
def getHadoopQAUser(cls): return Config.get('hadoop', 'HADOOPQA_USER')
def getCluster(cls): return Config.get('hadoop', 'CLUSTER')
def getMapredUser(cls): return Config.get('hadoop', 'MAPRED_USER')
def getHDFSUser(cls): return Config.get('hadoop', 'HDFS_USER')
def getHadoopConfDir(cls): return Config.get('hadoop', 'HADOOP_CONF')
def getLocalDirInfo(host): return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "mapred-site.xml"), "mapred.local.dir")
def runAgent(cls, name, conffile, user=None, cwd=None, env=None): flume_conf = Config.get('flume-ng', 'FLUME_CONF') cmd = "agent -n %s -c %s -f %s" % (name, flume_conf, conffile) return cls.runInBackgroundAs(user, cmd, cwd=cwd, env=env)
def getConfigValue(cls, propertyValue, defaultValue=None): return util.getPropertyValueFromConfigXMLFile(os.path.join(Config.get('hadoop', 'HADOOP_CONF'), "core-site.xml"), propertyValue, defaultValue=defaultValue)
def runInBackgroundAs(cls, user, cmd, cwd=None, env=None): flume_cmd = Config.get('flume-ng', 'FLUME_CMD') flume_cmd += " " + cmd return Machine.runinbackgroundAs(user, flume_cmd, cwd=cwd, env=env)
def getDatanodes(cls): slaveFile = os.path.join(Config.get('hadoop', 'HADOOP_CONF'),"slaves") f = open(slaveFile, "r") datanodes = f.readlines() return datanodes
def runas(cls, user, cmd, cwd=None, env=None, logoutput=True): flume_cmd = Config.get('flume-ng', 'FLUME_CMD') flume_cmd += " " + cmd return Machine.runas(user, flume_cmd, cwd=cwd, env=env, logoutput=logoutput)
def runas(cls, user, cmd, logoutput=True): hadoop_cmd = Config.get('hadoop', 'HADOOP_CMD') if Config.get('hadoop', 'HADOOP_CONF_EXCLUDE') == 'False': hadoop_cmd += " --config " + Config.get('hadoop', 'HADOOP_CONF') hadoop_cmd += " " + cmd return Machine.runas(user, hadoop_cmd, logoutput=logoutput)
def getVersion(cls): return Config.get('hadoop', 'VERSION')