def change_master_version(cls, version, config=None, logText="Upgrade"):
'''
Changes Master services:
:param version: Version to be changed to
:param config: Config location
'''
logger.info("Falcon - Begin function change_master_version ")
from beaver.component.rollingupgrade.ruCommon import hdpSelect
from time import sleep
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
node = Falcon.get_falcon_server()
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[INFO][Falcon][%s] Falcon server for node %s to version %s started"
% (logText, node, version))
Falcon.stop(cls._falcon_user, node)
sleep(30)
hdpSelect.changeVersion("falcon-server", version, node)
hdpSelect.changeVersion("falcon-client", version, node)
Falcon.start(cls._falcon_user, node)
sleep(60)
UpgradePerNode.reportProgress(
"[INFO][Falcon][%s] Falcon server for node %s to version %s finished"
% (logText, node, version))
return
def get_local_job_summary_logs(cls, component):
"""
Copy Job_summary Logs to local dirs [artifacts/job_summary_local.log]
parameter: component : Component name for which log collection is taking place
return: List of Local copies of Job summary log
Note: Some components need special handling where there are multiple Job Summary Log files
such as HA and Falcon
"""
LocalJobSummaryLogs = []
try:
if component == FALCON_STR:
from beaver.component.falcon import Falcon # pylint: disable=redefined-outer-name
host1 = Falcon.get_cluster_1_masters()['rm']
host2 = Falcon.get_cluster_2_masters()['rm']
host3 = Falcon.get_cluster_3_masters()['rm']
for host in [host1, host2, host3]:
JobSummaryLog = ComponentLogUtil.MAPRED_getJobSummaryLogFile(
host)
LocalJobSummaryLog = os.path.join(
Config.getEnv('ARTIFACTS_DIR'),
"jobsummary_" + host + ".log")
Machine.copyToLocal(None, host, JobSummaryLog,
LocalJobSummaryLog, None)
if Machine.pathExists(None,
None,
LocalJobSummaryLog,
passwd=None):
LocalJobSummaryLogs.append(LocalJobSummaryLog)
else:
for host in ComponentLogUtil.YARN_getRMHANodes():
JobSummaryLog = ComponentLogUtil.MAPRED_getJobSummaryLogFile(
host)
LocalJobSummaryLog = os.path.join(
Config.getEnv('ARTIFACTS_DIR'),
"jobsummary_" + host + ".log")
Machine.copyToLocal(Machine.getAdminUser(), host,
JobSummaryLog, LocalJobSummaryLog,
Machine.getAdminPasswd())
Machine.chmod("777",
LocalJobSummaryLog,
user=Machine.getAdminUser(),
passwd=Machine.getAdminPasswd())
if Machine.pathExists(Machine.getAdminUser(),
None,
LocalJobSummaryLog,
passwd=Machine.getAdminPasswd()):
LocalJobSummaryLogs.append(LocalJobSummaryLog)
return LocalJobSummaryLogs
except Exception as e:
logger.info("Exception occurs at job_summary_log collection %s", e)
tb = traceback.format_exc()
logger.info(tb)
return LocalJobSummaryLogs
def runCommand(cls, user, cmd, ignoreError=True):
"""
Function to run falcon commands cmd as a certain user
:param user:
:param cmd:
:return:
"""
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return 0, ""
exit_code, output = Falcon.runas(user, cmd)
if ignoreError:
if exit_code != 0:
#logger.info("Warning (ignoreError=True): Non-zero exit code when running command " + cmd + " as user " + user)
from beaver.component.rollingupgrade.ruUpgrade import UpgradePerNode
UpgradePerNode.reportProgress(
"[FAILED][Falcon] Warning (ignoreError=True): Non-zero exit code when running command %s as user %s "
% (cmd, user))
else:
ruAssert(
"Falcon", exit_code == 0,
"[RunCommand] Could not run command " + cmd + " as user " +
user)
return exit_code, output
def Falcon_getLogDir(cls, logoutput=True):
try:
from beaver.component.falcon import Falcon
return Falcon.getLogDir(logoutput)
except Exception as e:
if logoutput:
logger.error(
"Exception occured during Falcon_getLogDir() call: %s",
str(e))
return None
def collect_application_log_for_Falcon_locally(cls, JobSummaryLogfile,
appId, user):
'''
Collects application logs for Falcon component and save it in Local Dir with <appId>.log filename
'''
host = re.search("jobsummary_(.*).log", JobSummaryLogfile).group(1)
if not Machine.pathExists(
None, None, os.path.join(cls.LOCAL_TMP_APP_STORAGE, host),
None):
Machine.makedirs(None, None,
os.path.join(cls.LOCAL_TMP_APP_STORAGE, host),
None)
Machine.chmod("777",
os.path.join(cls.LOCAL_TMP_APP_STORAGE, host),
recursive=True,
user=None,
host=None,
passwd=None,
logoutput=True)
filename = os.path.join(cls.LOCAL_TMP_APP_STORAGE, host,
appId + ".log")
try:
from beaver.component.falcon import Falcon # pylint: disable=redefined-outer-name
Falcon.get_application_log(host,
appId,
appOwner=user,
nodeAddress=None,
containerId=None,
logoutput=False,
grepFilter=None,
pipeToFileOutput=filename)
except Exception:
logger.error(
"Exception occured during collect_application_log_for_Falcon_locally() call"
)
logger.error(traceback.format_exc())
logger.info("Get application log for Falcon is broken")
def createClusterEntities(cls, colo, desc, name):
try:
from beaver.component.falcon import Falcon
except ImportError:
## Import fails when Falcon is not installed on this machine. Nothing to do
return
from beaver.component.hadoop import Hadoop, HDFS, YARN
write_endpoint = Hadoop.getFSDefaultValue()
webhdfs_scheme = 'webhdfs'
if HDFS.isHttpsEnabled():
webhdfs_scheme = 'swebhdfs'
read_endpoint = '%s://%s:%s' % (
webhdfs_scheme, write_endpoint.split('/')[2].split(':')[0],
HDFS.getNNWebPort())
execute_endpoint = YARN.getResourceManager()
falconNode = Falcon.get_falcon_server()
from beaver.component.oozie import Oozie
oozieUrl = Oozie.getOozieUrl()
entityText = "<?xml version=\"1.0\"?>" \
"<cluster colo=\"" + colo + "\" description=\"" + desc + "\" name=\"" + name + "\" " \
"xmlns=\"uri:falcon:cluster:0.1\"> " \
"<interfaces> " \
"<interface type=\"readonly\" endpoint=\""+read_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"write\" endpoint=\""+write_endpoint+"\" version=\"0.20.2\"/> " \
"<interface type=\"execute\" endpoint=\"" + execute_endpoint + "\" version=\"0.20.2\"/> " \
"<interface type=\"workflow\" endpoint=\"" + oozieUrl + "\" version=\"3.1\"/>" \
"<interface type=\"messaging\" endpoint=\"" \
"tcp://" + falconNode + ":61616?daemon=true\" version=\"5.1.6\"/>" \
"</interfaces>" \
"<locations>" \
"<location name=\"staging\" path=\"/apps/falcon/" + name + "/staging\" />" \
"<location name=\"temp\" path=\"/tmp\" />" \
"<location name=\"working\" path=\"/apps/falcon/" + name + "/working\" />" \
"</locations>" \
"<ACL owner=\"" + cls._job_user + "\" group=\"users\" permission=\"0755\"/>"
if Hadoop.isSecure():
realm = HDFS.getConfigValue(
'dfs.namenode.kerberos.principal').split('@')[1]
entityText += "<properties> <property name=\"dfs.namenode.kerberos.principal\" value=\"nn/_HOST@" + realm + "\"/> </properties>"
entityText += "</cluster>"
textFile = open(os.path.join(cls._local_workspace, name + ".xml"), "w")
textFile.write("%s" % entityText)
textFile.close()
return