def queryserver(cls,
user,
host,
action,
config=None,
homePath=None,
binFolder='bin'):
if Machine.type() == 'Windows':
return Machine.service("queryserver", action, host=host)
else:
if homePath is None:
homePath = Config.get('phoenix', 'PHOENIX_HOME')
cmd = os.path.join(homePath, binFolder, 'queryserver.py')
env = None
if config:
env = {}
env['HBASE_CONF_DIR'] = config
cmd = "%s %s" % (cmd, action)
exit_code, stdout = Machine.runas(user, cmd, host=host, env=env)
# The command exited abnormally, don't run any follow-on checks
if exit_code != 0:
logger.warn("Failed to execute queryserver command. %d, %s",
exit_code, stdout)
return exit_code, stdout
# Check if PQS is accepting HTTP request before returning
if action == 'start':
logger.info('Verifying that PQS is running')
if Phoenix.verify_pqs_running(host, QUERYSERVER_PORT):
return exit_code, stdout
# Failed to verify PQS is running, bail out.
raise Exception('Failed to connect to PQS on %s' % host)
return exit_code, stdout
def setupHS2ConcurrencyDataset():
logger.info("Setup test data")
data_dir = os.path.join(Config.getEnv('ARTIFACTS_DIR'), "hs2concur-test-data")
data_tgz = os.path.join(Config.getEnv('WORKSPACE'), "hs2concur-test-data.tgz")
if not os.path.isfile(data_tgz):
assert util.downloadUrl(Config.get('hive', 'HS2CONCURR_TEST_DATA'), data_tgz)
Machine.tarExtractAll(data_tgz, data_dir)
# load data into HDFS
hdfs_user = Config.get("hadoop", 'HDFS_USER')
HDFS.createDirectory("/tmp/hs2data", user=hdfs_user, perm='777', force=True)
HDFS.createDirectory("/tmp/hs2data/student", perm='777', force=True)
HDFS.copyFromLocal(os.path.join(data_dir, 'studenttab10k'), "/tmp/hs2data/student")
HDFS.createDirectory("/tmp/hs2data/voter", perm='777', force=True)
HDFS.copyFromLocal(os.path.join(data_dir, 'votertab10k'), "/tmp/hs2data/voter")
query = """drop table if exists student_txt;
create external table student_txt (name string, age int, gpa double) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/student';
drop table if exists voter_txt;
create external table voter_txt (name string, age int, registration string, contributions float) row format delimited fields terminated by '\\t' stored as textfile location '/tmp/hs2data/voter';
drop table if exists student;
create table student (name string, age int, gpa double) CLUSTERED BY (name) INTO 20 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true');
drop table if exists voter;
create table voter (name string, age int, registration string, contributions float) CLUSTERED BY (name) INTO 20 BUCKETS STORED AS ORC TBLPROPERTIES('transactional'='true');
Insert into table student select * from student_txt;
Insert into table voter select * from voter_txt;"""
exit_code, stdout, stderr = Hive.runQueryOnBeeline(query, readFromFile=True, logoutput=True)
assert exit_code == 0, "Test data creation failed"
def HDFS_getNNLogDir(cls, logoutput=True):
try:
nnHostList = cls.HDFS_getNamenodes()
if nnHostList:
for nnHost in nnHostList:
lines = Machine.find(user=Machine.getAdminUser(),
host=nnHost,
filepath=Config.get(
'hadoop', 'HADOOP_LOG_DIR'),
searchstr="hadoop*-namenode-*.log",
passwd=Machine.getAdminPasswd())
lines = util.prune_output(lines, Machine.STRINGS_TO_IGNORE)
if lines:
nnLogFilePath = lines[0]
if nnLogFilePath is None:
continue
else:
return util.getDir(nnLogFilePath)
return None
except Exception:
if logoutput:
logger.error(
"Exception occured during HDFS_getNNLogDir() call")
logger.error(traceback.format_exc())
return None
def setupMondrianDataset():
DATABASE_NAME = 'foodmart'
LOCAL_DATA_DIR = os.path.join(Config.getEnv('ARTIFACTS_DIR'), DATABASE_NAME)
FOODMART_DDL = os.path.join(LOCAL_DATA_DIR, "foodmart.ddl")
HADOOPQA_USER = Config.get("hadoop", 'HADOOPQA_USER')
logger.info("Setup Mondrian dataset")
if not os.path.exists(LOCAL_DATA_DIR):
MONDRIAN_DATA_TGZ = LOCAL_DATA_DIR + ".tgz"
assert util.downloadUrl(Config.get('hive', 'MONDRIAN_DATASET'), MONDRIAN_DATA_TGZ)
Machine.tarExtractAll(MONDRIAN_DATA_TGZ, Config.getEnv('ARTIFACTS_DIR'))
assert os.path.isdir(LOCAL_DATA_DIR)
logger.info("create foodmart database and tables")
HDFS.createDirectory("/tmp/mondrian", HADOOPQA_USER, perm='777', force=True)
HDFS.copyFromLocal(LOCAL_DATA_DIR, "/tmp/mondrian", HADOOPQA_USER)
HDFS.chmod(None, 777, "/tmp/mondrian", recursive=True)
exit_code, stdout, stderr = Hive.runQueryOnBeeline(
FOODMART_DDL,
hivevar={
'DB': 'foodmart',
'LOCATION': '/tmp/mondrian/foodmart'
},
logoutput=True,
queryIsFile=True
)
assert exit_code == 0, "Unable to deploy foodmart dataset"
def importDmp(cls,
dmpfile,
host="",
fromuser="",
touser="",
touserpasswd="",
database='xe'):
cls.prepareEnv()
if Config.get("sqoop", "SQOOP_RUN_WITH_ORACLE12") == 'true':
oracleHome = "/u01/app/oracle/product/12.1.0/dbhome_1/"
oracleSid = "ORCL"
database = 'orcl'
ORCL_USER = "******"
if Machine.isEC2():
keypair = "/tmp/ec2-keypair"
else:
keypair = "/tmp/ycloud.pem"
# pylint: disable=line-too-long
run_cmd = "ssh -i %s -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t -t %s@%s 'sudo ORACLE_SID=%s ORACLE_HOME=%s %s/bin/imp %s/%s file=%s" % (
keypair, ORCL_USER, host, oracleSid, oracleHome, oracleHome,
touser, touserpasswd, dmpfile)
# pylint: enable=line-too-long
host = ""
else:
run_cmd = "/u01/app/oracle/product/11.2.0/xe/bin/imp %s/%s@%s file=%s" % (
touser, touserpasswd, database, dmpfile)
if fromuser != "":
run_cmd += " fromuser="******"":
run_cmd += " touser="******"sqoop", "SQOOP_RUN_WITH_ORACLE12") == 'true':
run_cmd += "'"
return Machine.runas(ROOT_USER, run_cmd, host)
def __init__(self, host='localhost', port=None, isHttps=False):
if port is None:
if (Hadoop.isEncrypted() or Ambari.is_ambari_encrypted() and Machine.isHumboldt() == False):
port = 8443
isHttps = True
else:
port = 8080
if isHttps or self.isCloudbreak():
self.baseUrl = 'https://' + host
else:
self.baseUrl = 'http://' + host
if self.isCloudbreak():
self.baseUrl = self.baseUrl + '/ambari'
else:
self.baseUrl = self.baseUrl + ':' + str(port)
if Machine.isHumboldt():
self.username_password = Config.get('ambari', 'AMBARI_USERNAME', 'admin') + ':HdpCli123!'
ambari_gateway = Config.get('machine', 'GATEWAY').replace("-ssh", "")
self.baseUrl = 'https://%s' % ambari_gateway
elif Machine.getInstaller() == 'cloudbreak':
self.username_password = Config.get('ambari', 'AMBARI_USERNAME', 'admin') + ':cloudbreak1'
else:
self.username_password = Config.get('ambari', 'AMBARI_USERNAME', 'admin'
) + ':' + Config.get('ambari', 'AMBARI_PASSWORD', 'admin')
self.urlLogin = self.baseUrl + '#/login'
self.urlGetClusters = self.baseUrl + '/api/v1/clusters'
self.urlGetAmbClusters = self.baseUrl + '/api/v1/services'
self.urlConfig = '/configurations'
self.backupDataJson = dict()
self.logger = logging.getLogger(__name__)
def getVersion(cls):
'''
Returns Knox version
'''
KNOX_VERSION_CMD = os.path.join(KNOX_HOME, "bin",
cls.getKnoxCli()) + " version"
VERSION_CMD = "export JAVA_HOME=%s; %s" % (JAVA_HOME, KNOX_VERSION_CMD)
if Machine.type() == 'Windows':
VERSION_CMD = KNOX_VERSION_CMD
exit_code, stdout = Machine.runas(Machine.getAdminUser(),
VERSION_CMD,
host=KNOX_HOST,
env=None,
logoutput=True)
logger.info("stdout = " + stdout)
if exit_code == 0:
pattern = re.compile("Apache Knox: (\S+)")
m = pattern.search(stdout)
if m:
cls._version = m.group(1)
else:
cls._version = ""
else:
cls._version = ""
return cls._version
def getZooLogDir(cls, logoutput=False):
'''
Returns Zookeeper log directory (String).
'''
matchObjList = None
if Machine.isHumboldt():
try:
from beaver.component.hbase import HBase
#get some zookeeper node
hmaster_nodes = HBase.getAllMasterNodes()
if hmaster_nodes:
zkNode = hmaster_nodes[0]
if zkNode:
Machine.copyToLocal(None, zkNode, ZOOKEEPER_ENV_FILE,
Machine.getTempDir())
REMOTE_ZOOKEEPER_ENV_FILE = os.path.join(
Machine.getTempDir(), 'zookeeper-env.sh')
matchObjList = util.findMatchingPatternInFile(
REMOTE_ZOOKEEPER_ENV_FILE,
"export ZOO_LOG_DIR=(.*)",
return0Or1=False)
except Exception:
pass
if not matchObjList:
#gateway should have the config file.
matchObjList = util.findMatchingPatternInFile(
ZOOKEEPER_ENV_FILE,
"export ZOO_LOG_DIR=(.*)",
return0Or1=False)
returnValue = None
if matchObjList:
returnValue = matchObjList[0].group(1)
if logoutput:
logger.info("Zookeeper.getZooLogDir returns %s", returnValue)
return returnValue
def getZKpid(cls, host=None):
'''
Find zookeeper pid from machine
:param host: machine hostname
:return: pid of zookeeper service
'''
service = "zookeeper"
if Machine.isWindows():
pid = Machine.getProcessListWithPid(host,
"java.exe",
service,
logoutput=True,
ignorecase=True)
return int(pid)
else:
pids = Machine.getProcessListRemote(host,
format="%U %p %P %a",
filter=service,
user=None,
logoutput=True)
for p in pids:
pid = Machine.getPidFromString(p,
user=None,
isPidStringWithoutuser=True)
if not pid:
# TODO: Need to convert into raw string to fix pylint issue. Disabled it temporarily.
pid = Machine.getPidFromString(
p, user="******", isPidStringWithoutuser=False) # pylint: disable=anomalous-backslash-in-string
if pid:
return int(pid)
return None
def read_and_update_job_parameters(self):
print "COMPONENT = ", self.COMPONENT
print "TESTSUITE_FILE = ", self.TESTSUITE_FILE
print "SRC_DIR = ", self.SRC_DIR
print "LOCAL_WORK_DIR = ", self.LOCAL_WORK_DIR
#Set permissions on ambari testcode location
Machine.runas('root', 'chmod -R 755 ' + self.DEPLOY_CODE_DIR)
#copy the ambari ui test code to artifacts dir
shutil.copytree(self.SRC_DIR, self.LOCAL_WORK_DIR)
#copy the ambari pytest file to artifacts dir
shutil.copy(self.PYTEST_FILE, self.LOCAL_WORK_DIR)
#populate the properties file
propChanges = CommonLib.read_job_properties(self.LOCAL_WORK_DIR)
self.splitNumStr = str(propChanges['SPLIT_NUM'])
CommonLib.log_test_properties(propChanges)
CommonLib.delete_test_time_duration_file()
self.env['DISPLAY'] = self.DISPLAY
CommonLib.common_startup_operations(self.LOCAL_WORK_DIR)
def kinitas(cls,
keytabUser,
principal=None,
keytabFile=None,
flag="-f",
logoutput=True,
host=None):
'''
Runs kinit as specified keytab user.
Returns (exit_code, stdout).
'''
# Build common variables and cache them.
cls._buildCommonVar()
# If keytabUser is None, use current user
if keytabUser is None:
keytabUser = Config.getEnv('USER')
# Get kerberos ticket location e.g. /grid/0/hadoopqe/artifacts/kerberosTickets/hrt_qa.kerberos.ticket
kerbTicket = Machine.getKerberosTicket(user=keytabUser)
# If keytab is unset, use default keytab path e.g. /home/hrt_qa/hadoopqa/keytabs/hrt_qa.headless.keytab
if keytabFile is None:
keytabFile = Machine.getHeadlessUserKeytab(keytabUser)
# If principal is not set, use keytab user.
if principal is None:
principal = keytabUser
# Build command
cmd = "%s -c %s -k -t %s %s %s" % (
cls._kinitloc, kerbTicket, keytabFile, flag,
Machine.get_user_principal(principal))
# kinit always runs with credential of current user.
return Machine.runas(user=None,
cmd=cmd,
env=None,
logoutput=logoutput,
host=host)
def background_job_setup(cls, runSmokeTestSetup=True, config=None):
'''
Create 5 input datasets for TestOrderedWordCount
:param runSmokeTestSetup: Runs smoke test setup if set to true
'''
logger.info("*** Start background job setup for Tez ***")
Machine.rm(user=HADOOPQA_USER,
host=None,
filepath=LOCAL_WORK_DIR,
isdir=True)
os.mkdir(LOCAL_WORK_DIR)
for i in range(0, 4, 1):
inputDirName = "HDFS_INPUT%d" % i
inputDirPath = os.path.join(LOCAL_WORK_DIR, inputDirName)
HadoopJobHelper.runCustomWordWriter(LOCAL_WORK_DIR, inputDirPath,
10, 400, 10000)
hdfsInputDir = "/user/%s/Input%d" % (HADOOPQA_USER, i)
hdfsOutputDir = "/user/%s/output%d" % (HADOOPQA_USER, i)
#In case already present, delete the input directory
HDFS.deleteDirectory(hdfsInputDir)
HDFS.createDirectory(hdfsInputDir)
HDFS.deleteDirectory(hdfsOutputDir)
HDFS.copyFromLocal(inputDirPath, hdfsInputDir)
cls._hdfsInputList.append(hdfsInputDir + "/" + inputDirName)
cls._hdfsOutputList.append(hdfsOutputDir)
logger.info("Created data for input %d", i)
logger.info("*** End background job setup for Tez ***")
def _addOrganizationUnit(cls):
if not os.path.isdir(cls._tmp_path):
Machine.makedirs(None, None, cls._tmp_path)
tmp_path = os.path.join(cls._tmp_path, "users.ldif")
file_ = open(tmp_path, 'w')
file_.write('dn: ou=Users,dc=hortonworks,dc=com\n')
file_.write('objectClass: organizationalUnit\n')
file_.write('ou: Users\n')
file_.close()
command = "ldapadd -x -f " + tmp_path + " -D cn=Manager,dc=hortonworks,dc=com -w password"
exit_code, stdout = Machine.runas("hrt_qa", command, None, None, None, "True", None)
# the below if statement takes care of a case when adding item already exists
if stdout.find("Already exists") == -1:
assert exit_code == 0, "Unable to Users add organization unit"
tmp_path = os.path.join(cls._tmp_path, "groups.ldif")
file_ = open(tmp_path, 'w')
file_.write('dn: ou=Groups,dc=hortonworks,dc=com\n')
file_.write('objectClass: organizationalUnit\n')
file_.write('ou: Groups\n')
file_.close()
command = "ldapadd -x -f " + tmp_path + " -D cn=Manager,dc=hortonworks,dc=com -w password"
exit_code, stdout = Machine.runas("hrt_qa", command, None, None, None, "True", None)
# the below if statement takes care of a case when adding item already exists
if stdout.find("Already exists") == -1:
assert exit_code == 0, "Unable to add Groups organization unit"
def http_put_post_request(self, url, data, method):
#self.logger.info(url)
if (Machine.isHumboldt()):
basic_auth = HTTPBasicAuth('admin', 'HdpCli123!')
elif Machine.getInstaller() == 'cloudbreak':
basic_auth = HTTPBasicAuth('admin', 'cloudbreak1')
else:
basic_auth = HTTPBasicAuth('admin', 'admin')
if method == 'put':
response = requests.put(
url=url, data=data, auth=basic_auth, headers={'X-Requested-By': 'ambari'}, verify=False
)
return response
elif method == 'post':
response = requests.post(
url=url, data=data, auth=basic_auth, headers={'X-Requested-By': 'ambari'}, verify=False
)
return response
elif method == 'get':
response = requests.get(
url=url, data=data, auth=basic_auth, headers={'X-Requested-By': 'ambari'}, verify=False
)
return response
else:
return None
def runCmd( # pylint: disable=unused-argument
cls,
cmd,
host="",
database="",
user="",
passwd="",
cwd=None,
wallet=False):
cls.prepareEnv()
if database == "":
database = Config.get('machine', 'ORACLE_DB', default='xe')
if wallet is False:
run_cmd = Machine.echocmd(
cmd) + " | sqlplus %s/%s@%s" % (user, passwd, database)
if Config.get("sqoop", "SQOOP_RUN_WITH_ORACLE12") == 'true':
oracleHome = "/u01/app/oracle/product/12.1.0/dbhome_1/"
database = 'orcl'
ORCL_USER = "******"
if Machine.isEC2():
keypair = "/tmp/ec2-keypair"
else:
keypair = "/tmp/ycloud.pem"
# pylint: disable=line-too-long
run_cmd = "ssh -i %s -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t -t %s@%s 'echo exit|sudo ORACLE_SID=ORCL ORACLE_HOME=%s %s/bin/sqlplus %s/%s %s'" % (
keypair, ORCL_USER, host, oracleHome, oracleHome, user,
passwd, cmd)
# pylint: enable=line-too-long
host = ""
else:
run_cmd = Machine.echocmd(cmd) + " | sqlplus /@w_orcl"
return Machine.runas(ROOT_USER, run_cmd, host, doEscapeQuote=False)
def Maven2runas(cls, cmd, local_work_dir, cwd=None, env=None, mavenOpts=None, logoutput=True, user=None):
from beaver.component.ambariHelper import Maven2
# make sure maven is setup before its run
Maven2.setupMaven()
# initialize env
if not env:
env = {}
# determine if MAVEN_OPTS need to be set
if mavenOpts:
opts = os.environ.get('MAVEN_OPTS')
if not opts:
opts = mavenOpts
else:
opts = ' '.join([opts, mavenOpts])
env['MAVEN_OPTS'] = opts
env['JAVA_HOME'] = Maven2._java_home
env['M2_HOME'] = Maven2._maven_home
# print the env so we can see what we are setting
logger.info('Env for mvn cmd')
logger.info(env)
print "Maven2runas:LOCAL_WORK_DIR = ", local_work_dir
Machine.runas('root', 'chmod -R 777 ' + local_work_dir)
maven_cmd = "cd %s; %s %s" % (local_work_dir, Maven2._maven_cmd, cmd)
exit_code, stdout = Machine.runas(user, maven_cmd, cwd=cwd, env=env, logoutput=logoutput)
return exit_code, stdout
def setupAcidDataset(testsuite, LOCAL_DIR):
ddl_location = None
if testsuite == 'acid':
ddl_location = os.path.join(LOCAL_DIR, "ddl", "acid-tpch-tablesetup.sql")
elif testsuite == 'unbucketed':
ddl_location = os.path.join(LOCAL_DIR, "ddl", "acid-tpch-unbucketed-tablesetup.sql")
else:
assert 1 == 0, "The testsuite passed in not correct. Please use value 'acid' or 'unbuckted'"
# change timezone on test machines
Machine.resetTimeZoneOnCluster()
# Download TPCH acids data
tpch_newdata_dir = os.path.join(LOCAL_DIR, "tpch_newdata_5G")
TPCH_STAGE_TGZ = os.path.join(LOCAL_DIR, "tpch_newdata_5G.tgz")
if not os.path.isfile(TPCH_STAGE_TGZ):
assert util.downloadUrl(Config.get('hive', 'TPCH_NEWDATA_5G_DNLD_URL'), TPCH_STAGE_TGZ)
Machine.tarExtractAll(TPCH_STAGE_TGZ, LOCAL_DIR)
# Load the acid tables in Hive
HADOOPQA_USER = Config.get("hadoop", 'HADOOPQA_USER')
HDFS.createDirectory("/tmp/lineitem_acid", user=HADOOPQA_USER, perm='777', force=True)
HDFS.copyFromLocal(os.path.join(tpch_newdata_dir, "lineitem*"), "/tmp/lineitem_acid", HADOOPQA_USER)
HDFS.chmod(None, 777, "/tmp/lineitem_acid", recursive=True)
exit_code, stdout, stderr = Hive.runQueryOnBeeline(
ddl_location, hivevar={'HDFS_LOCATION': '/tmp'}, logoutput=True, queryIsFile=True
)
assert exit_code == 0, "Failed to populate the TPCH acid data in Hive"
def MAPRED_getJobSummaryLogFile(cls, jtHost=None, logoutput=True):
if not jtHost:
jtHost = cls.MAPRED_getJobtracker()
try:
file_paths = [
Config.get('hadoop', 'YARN_LOG_DIR'),
Config.get('hadoop', 'HADOOP_LOG_DIR')
]
file_name = 'hadoop-mapreduce.jobsummary.log'
if Machine.isWindows():
file_name = 'rm-appsummary.log'
for file_path in file_paths:
lines = Machine.find(user=Machine.getAdminUser(),
host=jtHost,
filepath=file_path,
searchstr=file_name,
passwd=Machine.getAdminPasswd())
lines = util.prune_output(lines, Machine.STRINGS_TO_IGNORE)
if lines and lines[0]:
return lines[0]
else:
continue
# otherwise return the RM/jobtracker log
from beaver.component.hadoop import MAPRED
return MAPRED.getJobTrackerLogFile()
except Exception:
if logoutput:
logger.error(
"Exception occured during MAPRED_getJobSummaryLogFile() call"
)
logger.error(traceback.format_exc())
return None
def runas(cls,
user,
cmd,
cwd=None,
env=None,
logoutput=True,
runInBackground=False):
runCmd = Config.get('pig', 'PIG_CMD') + " " + cmd
# initialize env
if not env:
env = {}
# get kerberos ticket
if Hadoop.isSecure():
if user is None:
user = Config.getEnv('USER')
kerbTicket = Machine.getKerberosTicket(user)
env['KRB5CCNAME'] = kerbTicket
user = None
if runInBackground:
return Machine.runinbackgroundAs(user, runCmd, cwd=cwd, env=env)
else:
return Machine.runas(user,
runCmd,
cwd=cwd,
env=env,
logoutput=logoutput)
def get_hadoop_test_jar(cls):
return Machine.find(
user=Machine.getAdminUser(),
host='',
filepath=Config.get('hadoop', 'MAPRED_HOME'),
searchstr='hadoop-mapreduce-client-jobclient-*-tests.jar',
passwd=Machine.getAdminPasswd())[0]
def ru_rollback_state(cls):
'''
Saved state is rolled back - upgrade is abandonded
NOTE: this command will not return until namenode shuts down
:return:
'''
logger.info("[INFO][HDFS][Upgrade] HA Journal Node Upgrade Started ")
hdfs_user = Config.get('hadoop', 'HDFS_USER')
nodes = []
nodes.append(HDFS.getNamenodeByState('standby'))
nodes.append(HDFS.getNamenodeByState('active'))
logger.info("[INFO][HDFS][Upgrade] HA Namenode Upgrade Started")
for node in nodes:
HDFS.resetZkfc('stop', hosts=node.split())
HDFS.resetNamenode('stop', host=node)
HDFS.resetNamenode('start',
config=None,
host=nodes[0],
option=" -rollingUpgrade rollback")
HDFS.resetZkfc('start', hosts=nodes[0].split())
# lets make sure the NN is out of safemode before we proceed to the next namenode
HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[0])
command = "sudo su - -c 'hadoop namenode -bootstrapStandby -force' hdfs"
(exitcode, stdout) = Machine.runas(Machine.getAdminUser(), command,
nodes[1], None, None, "True",
Machine.getAdminPasswd())
ruAssert("HDFS", exitcode == 0,
"hadoop namenode -bootstrapStandby -force")
HDFS.resetNamenode('start', config=None, host=nodes[1], option="")
HDFS.resetZkfc('start', hosts=nodes[1].split())
# lets make sure the NN is out of safemode before we proceed to the next namenode
HDFS.waitForNNOutOfSafemode(options='-fs hdfs://%s:8020' % nodes[1])
def stop(cls):
cls.setupElasticSearch()
if os.path.isdir(cls._es_home):
pid = Machine.getPIDByPort(cls._es_transport_port)
if pid:
Machine.killProcess(pid)
time.sleep(2)
def getKerbPrincipalfromKeyTab(cls, keytab, cluster=None):
if cluster == None:
cluster = 'cl1'
cmd = "%s -k -t %s " % ("klist", keytab)
exit_code, output = Machine.runas(Machine.getAdminUser(), cmd, passwd=Machine.getAdminPasswd())
kerb_principal = str(((output.split(os.linesep)[-1]).split(' ')[-1]))
logger.info("Principal is " + kerb_principal)
return kerb_principal.strip()
def start(cls):
cls.setupSolr()
# Start the server with setting not to persist data between restarts
cmd = os.path.join(cls._javahome, "bin", "java") + " -jar start.jar"
Machine.runinbackground(cmd,
cwd=cls._solr_home,
env={'JAVA_HOME': cls._javahome})
assert util.waitForPortToOpen(Machine.getfqdn(), cls._solr_http_port)
def runSysCmd(cls, cmd, oracle_host, user=ROOT_USER):
if Machine.isEC2():
keypair = "/tmp/ec2-keypair"
else:
keypair = "/tmp/ycloud.pem"
run_cmd = "ssh -i %s -q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t -t %s@%s 'sudo %s'" % (
keypair, user, oracle_host, cmd)
return Machine.runas(ROOT_USER, run_cmd)
def get_gateway(cls):
if cls._gwHost == "localhost":
if Machine.isWindows():
# always use short hostname in Windows
return Machine.getfqdn(name="")
else:
cls._gwHost = socket.gethostbyname(socket.gethostname())
return cls._gwHost
def setup_storm_hdfs_topology(cls, useStandaloneCmd):
storm_version = Storm.getVersion(useStandaloneCmd=True)
try:
file_obj = open(HDFS_CONFIG_FILE, 'w')
if Hadoop.isSecure():
file_obj.write('hdfs.keytab.file: ' +
Machine.getHeadlessUserKeytab(
user=HADOOPQA_USER) + '\n')
file_obj.write('hdfs.kerberos.principal: ' +
Machine.get_user_principal(user=HADOOPQA_USER) +
'\n')
finally:
file_obj.close()
HDFS.createDirectory("/tmp/mySeqTopology", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/dest", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/dest2", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/foo", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/trident", HDFS_USER, "777", False)
HDFS.createDirectory("/tmp/trident-seq", HDFS_USER, "777", False)
Machine.copy(JAVA_HDFS_SRC_DIR,
LOCAL_HDFS_WORK_DIR,
user=None,
passwd=None)
if not Machine.isWindows():
(exit_code, _) = Maven.run('package',
cwd=LOCAL_HDFS_WORK_DIR,
env={
HADOOP_VERSION_MAVEN_PARAMETER:
HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER:
storm_version,
HADOOP_CONF_MAVEN_PARAMETER:
HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER:
HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER:
HADOOP_CONF,
CORE_FILE_MAVEN_PARAMETER:
CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER:
Maven.getPublicRepoUrl()
})
else:
filepath = os.path.join(MOD_CONF_PATH, "core-site.xml")
(exit_code, _) = Maven.run(
'package -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s -D%s=%s'
% (HADOOP_VERSION_MAVEN_PARAMETER, HADOOP_VERSION,
STORM_VERSION_MAVEN_PARAMETER, storm_version,
HADOOP_CONF_MAVEN_PARAMETER, HADOOP_CONF,
HDFS_FILE_MAVEN_PARAMETER, HDFS_FILE,
HADOOP_CORE_MAVEN_PARAMETER, HADOOP_CONF,
CORE_FILE_MAVEN_PARAMETER, CORE_FILE,
PUBLIC_REPO_MAVEN_PARAMETER, Maven.getPublicRepoUrl()),
cwd=LOCAL_HDFS_WORK_DIR)
ruAssert("Storm", exit_code == 0,
"[StormHDFSSetup] maven package command failed")
def background_job_teardown(cls):
'''
Cleanup of HDFS background job
'''
HDFS.deleteDirectory(cls._base_hdfs_dir)
command = "rm -rf " + cls._lgStructureDir
exit_code, stdout = Machine.runas(Machine.getAdminUser(), command,
None, None, None, "True",
Machine.getAdminPasswd())
def runCmd(cls, cmd, host="", database="", user="", passwd="", cwd=None):
if passwd == "":
return Machine.run("mysql -h%s -u%s -e \"%s\" %s" %
(host, user, cmd, database),
cwd=cwd)
else:
return Machine.run("mysql -h%s -u%s -p%s -e \"%s\" %s" %
(host, user, passwd, cmd, database),
cwd=cwd)
def getLdapStatus(cls):
if Machine.type() == 'Windows':
return cls.__runServiceScript__(operation="query",
knoxService="ldap",
serUser=Machine.getAdminUser())
else:
return cls.__runServiceScript__(operation="status",
knoxService="ldap",
serUser=Machine.getAdminUser())
def checkJobCreatedTempFileInTT(logFileDir, currentUser, currentJobId, currentAttemptId, logfile, taskTrackerHost):
pathFile = os.path.join(logFileDir, 'taskTracker', currentUser, 'jobcache', currentJobId, currentAttemptId, 'work', logfile)
logger.info("path file: " + pathFile)
result = False
if platform.system() == 'Windows':
result = os.path.isfile(pathFile)
else:
if CommonHadoopEnv.getIsSecure():
result = os.path.isfile(pathFile)
else:
cmd = "ls %s" % pathFile
sudocmd = Machine.sudocmd(cmd,MAPRED_USER)
sudocmd += "|wc -l"
logger.info("sudocmd = " + sudocmd)
out = Machine.run(sudocmd)
result = (out[0] == 0 and out[1] == "1")
return result
def runShellCmds(cls, cmds):
tfpath = getTempFilepath()
tf = open(tfpath, 'w')
for cmd in cmds:
tf.write(cmd + "\n")
tf.write("exit\n")
tf.close()
output = Machine.run("hbase shell %s" % tfpath)
os.remove(tfpath)
return output
def run(cls, cmd, logoutput=True):
return Machine.run(Config.get('hbase', 'HBASE_CMD') + " " + cmd, logoutput=logoutput)
def runInBackgroundAs(cls, user, cmd):
hadoop_cmd = Config.get('hadoop', 'HADOOP_CMD')
if Config.get('hadoop', 'HADOOP_CONF_EXCLUDE') == 'False':
hadoop_cmd += " --config " + Config.get('hadoop', 'HADOOP_CONF')
hadoop_cmd += " " + cmd
return Machine.runinbackgroundAs(user, hadoop_cmd)
def runInBackgroundAs(cls, user, cmd, cwd=None, env=None):
flume_cmd = Config.get('flume-ng', 'FLUME_CMD')
flume_cmd += " " + cmd
return Machine.runinbackgroundAs(user, flume_cmd, cwd=cwd, env=env)
def runas(cls, user, cmd, cwd=None, env=None, logoutput=True):
flume_cmd = Config.get('flume-ng', 'FLUME_CMD')
flume_cmd += " " + cmd
return Machine.runas(user, flume_cmd, cwd=cwd, env=env, logoutput=logoutput)
def resetDatanode(cls,action):
datanodes = cls.getDatanodes()
Machine.resetNode("datanode", datanodes, action)
def resetTasktracker(cls,action):
tasktrackers = cls.getTasktrackers()
Machine.resetNode("tasktracker", tasktrackers, action)
def resetJobtracker(cls,action):
jobtracker = MAPRED.getJobTracker().split()
Machine.resetNode("jobtracker", jobtracker, action)
def resetNamenode(cls,action):
namenode = HDFS.getNamenode().split()
Machine.resetNode("namenode", namenode, action)
def runas(cls, user, cmd, logoutput=True):
hadoop_cmd = Config.get('hadoop', 'HADOOP_CMD')
if Config.get('hadoop', 'HADOOP_CONF_EXCLUDE') == 'False':
hadoop_cmd += " --config " + Config.get('hadoop', 'HADOOP_CONF')
hadoop_cmd += " " + cmd
return Machine.runas(user, hadoop_cmd, logoutput=logoutput)
