def start(cls, host, hcat_confdir=None):
if Machine.type() == 'Windows':
if not hcat_confdir == None:
HCAT_CONF_DIR = Config.get('hcatalog', 'HCATALOG_CONF')
Machine.copyFromLocal(Machine.getAdminUser(),
host,
os.path.join(hcat_confdir,
'webhcat-site.xml'),
HCAT_CONF_DIR,
passwd=Machine.getAdminPasswd())
Machine.service("templeton", "start", host=host)
else:
env = {}
if not hcat_confdir == None:
env['WEBHCAT_CONF_DIR'] = hcat_confdir
hcat_user = Config.get('hcatalog', 'HCATALOG_USER', 'hcat')
hcat_bin = os.path.join(
Config.get('hcatalog', 'HCATALOG_HOME',
'/usr/hdp/current/hive-webhcat'), 'sbin')
start_cmd = os.path.join(hcat_bin, 'webhcat_server.sh') + ' start'
exit_code, stdout = Machine.runas(hcat_user,
start_cmd,
host,
env=env,
passwd=Machine.getAdminPasswd())
assert exit_code == 0, "Unable to start webhcat_server.sh on %s \n %s " % (
host, start_cmd)
def deleteConfig( # pylint: disable=redefined-builtin
changes,
confDir,
updatedConfDir,
nodes,
isFirstUpdate=True,
id=None):
backuploc = getBackupConfigLocation(id=id)
if Machine.type() == 'Windows':
# clean up the backup
Machine.rm(None, Machine.getfqdn(), backuploc, isdir=True, passwd=None)
util.copyDir(confDir, backuploc)
if isFirstUpdate:
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
if isFirstUpdate:
util.copyDir(confDir, tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file: %s", filepath)
#only supports xml file for now
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
util.deletePropertyFromXML(filepath, filepath, values)
elif fext == ".properties":
logger.info(
".properties file not supported for deleting config")
elif fext == ".sh":
logger.info(".sh format not supported for deleting config")
elif fext is None or fext == "" or fext == ".include":
logger.info(
".incluse format not supported for deleting config")
# in windows world copy the configs back to the src location
if Machine.type() == 'Windows':
for node in nodes:
for filename in changes.keys():
Machine.copyFromLocal(None,
node,
os.path.join(tmpConfDir, filename),
os.path.join(confDir, filename),
passwd=None)
else:
for node in nodes:
Machine.rm(None, node, updatedConfDir, isdir=True)
Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def modifyConfigChaosMonkey(changes,
confDir,
updatedConfDir,
nodes,
isFirstUpdate=True):
if Machine.type() == 'Windows':
# clean up the backup
Machine.rm(None,
Machine.getfqdn(),
BACKUP_CONFIG_LOCATION,
isdir=True,
passwd=None)
util.copyDir(confDir, BACKUP_CONFIG_LOCATION)
if isFirstUpdate:
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
if isFirstUpdate:
util.copyDir(confDir, tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file: %s", filepath)
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
lines = file(filepath, 'r').readlines()
del lines[-1]
file(filepath, 'w').writelines(lines)
logger.info("Modifying file: %s", filepath)
util.writeToFile(values, filepath, isAppend=True)
# in windows world copy the configs back to the src location
if Machine.type() == 'Windows':
for node in nodes:
for filename in changes.keys():
Machine.copyFromLocal(None,
node,
os.path.join(tmpConfDir, filename),
os.path.join(confDir, filename),
passwd=None)
else:
for node in nodes:
Machine.rm(None, node, updatedConfDir, isdir=True)
Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def restoreConfig(changes, restoreLocation, nodes, id=None): # pylint: disable=redefined-builtin
'''
Restores hadoop config or config with similar structure.
Return None.
Linux: do nothing because default config is not altered.
Windows: copy config from BACKUP_CONFIG_LOCATION back to default config directories in target machines.
'''
backuploc = getBackupConfigLocation(id=id)
if Machine.type() != 'Windows':
return
for node in nodes:
for filename in changes:
Machine.copyFromLocal(None,
node,
os.path.join(backuploc, filename),
os.path.join(restoreLocation, filename),
passwd=None)
def runas(cls,
user,
cmd,
host="",
cwd=None,
env=None,
logoutput=True,
passwd=None):
if Machine.isLinux():
if user is None and passwd is None:
user = Machine.getAdminUser()
keypairLocation = "/home/hrt_qa/.ssh/id_rsa_log_server_hrt_qa"
if Machine.isHumboldt():
# for secure cluster in humboldt path is different
if not os.path.exists(keypairLocation):
keypairLocation = "/home/HDINSIGHT/hrt_qa/.ssh/id_rsa_log_server_hrt_qa"
# pylint: disable=line-too-long
cmd = "rsync -e 'ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s -o PreferredAuthentications=publickey' %s" % (
keypairLocation, ssh_port_number, cmd)
# pylint: enable=line-too-long
else:
# pylint: disable=line-too-long
cmd = "rsync -e \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s -o PreferredAuthentications=publickey\" %s" % (
ssh_port_number, cmd)
# pylint: enable=line-too-long
if logoutput:
logger.info("RSync.runas cmd=%s", cmd)
return Machine.runas(user, cmd, host, cwd, env, logoutput, passwd,
retry_count, retry_sleep_time)
else:
rsyncHome = "/cygdrive/c/testtools/cwrsync"
rsyncLocation = "c:\\testtools\\cwrsync\\rsync.exe"
#must use c:\\ so remote powershell can pick this up.
keypairLocation = "c:\\testtools\\id_rsa_log_server_hrt_qa"
# pylint: disable=line-too-long
cmd = "%s -e \"%s/ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s -o PreferredAuthentications=publickey\" %s" % \
(rsyncLocation, rsyncHome, keypairLocation, ssh_port_number, cmd)
# pylint: enable=line-too-long
if logoutput:
logger.info("RSync.runas cmd=%s", cmd)
if Machine.isSameHost(host, None):
pass
else:
# remote
# I cant find a way to call this on the fly.
# The only way it can work is to create bat file for the command and run the file.
batchFilename = 'rsync-%s.bat' % str(int(time.time()))
tmpLocalFile = os.path.join(Config.getEnv('ARTIFACTS_DIR'),
batchFilename)
logger.info("write to %s. contents=%s", tmpLocalFile, cmd)
util.writeToFile(cmd, tmpLocalFile)
destPath = os.path.join(Machine.getTempDir(), batchFilename)
Machine.copyFromLocal(Machine.getAdminUser(), host,
tmpLocalFile, destPath,
Machine.getAdminPasswd())
cmd = destPath
return Machine.runas(Machine.getAdminUser(),
cmd,
host,
cwd,
env,
logoutput,
passwd=Machine.getAdminPasswd(),
retry_count=retry_count,
retry_sleep_time=retry_sleep_time)
def modifyConfig( # pylint: disable=redefined-builtin
changes,
confDir,
updatedConfDir,
nodes,
isFirstUpdate=True,
makeCurrConfBackupInWindows=True,
id=None):
'''
Modifies hadoop config or config with similar structure.
Returns None.
Linux:
1. Create tmpModifyConfDir_<time> in artifacts dir based on source config directory in gateway
2. Modify contents in created directory.
3. Copy the directory to /tmp/hadoopConf in target machines
Windows:
1. If makeCurrConfBackupInWindows is True, backup current config first.
Copy current config to artifacts/HDPStackBackupConfig
2. Create tmpModifyConfDir_<time> in gateway.
3. Modify contents in created directory.
4. Copy the directory to target machines. Replace config in default locations in remote machines.
Calling modifyConfig twice, changes will be cumulative.
'''
backuploc = getBackupConfigLocation(id=id)
if Machine.type() == 'Windows' and makeCurrConfBackupInWindows:
# clean up the backup
Machine.rm(None, Machine.getfqdn(), backuploc, isdir=True, passwd=None)
util.copyReadableFilesFromDir(confDir, backuploc)
if isFirstUpdate:
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
if isFirstUpdate:
util.copyReadableFilesFromDir(confDir, tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file: %s", filepath)
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
util.writePropertiesToConfigXMLFile(filepath, filepath, values)
elif fext == ".json":
util.writePropertiesToConfigJSONFile(filepath, filepath,
values, ["global"],
"site.hbase-site.")
elif fext == ".properties":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".cfg":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".conf":
util.writePropertiesToConfFile(filepath, filepath, values)
elif fext == ".ini":
# 'shiro.ini : {'section:prop' : 'val}
util.writePropertiesToIniFile(filepath, filepath, values)
elif fext == ".sh":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".yaml":
text = ""
for k, v in values.iteritems():
text += k + " : " + v
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".cmd":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext is None or fext == "" or fext == ".include":
text = ""
isFirst = True
for value in values:
if isFirst:
text += value
else:
text += "\n" + value
isFirst = False
util.writeToFile(text, filepath, isAppend=True)
# in windows world copy the configs back to the src location
if Machine.type() == 'Windows':
for node in nodes:
for filename in changes.keys():
Machine.copyFromLocal(None,
node,
os.path.join(tmpConfDir, filename),
os.path.join(confDir, filename),
passwd=None)
else:
for node in nodes:
Machine.rm(user=Machine.getAdminUser(),
host=node,
filepath=updatedConfDir,
isdir=True,
passwd=Machine.getAdminPasswd())
Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def modifyConfigRemote(changes, OriginalConfDir, ConfDir, nodes, id=None): # pylint: disable=redefined-builtin
'''
Modifies hadoop config or config with similar structure.
Returns None.
Linux:
1. Create tmpModifyConfDir_<time> in artifacts dir based on source config directory in gateway
2. Modify contents in created directory.
3. Copy the directory to /tmp/hadoopConf in target machines
'''
_backuploc = getBackupConfigLocation(id=id)
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
for node in nodes:
Machine.rm(Machine.getAdminUser(), node, ConfDir, isdir=True)
Machine.rm(Machine.getAdminUser(),
Machine.getfqdn(),
tmpConfDir,
isdir=True)
logger.info("*** COPY ORIGINAL CONFIGS FROM REMOTE TO LOCAL ***")
Machine.copyToLocal(None, node, OriginalConfDir, tmpConfDir)
#if node == Machine.getfqdn():
# Machine.copy(OriginalConfDir,tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file locally: %s", filepath)
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
util.writePropertiesToConfigXMLFile(
filepath, filepath, values)
elif fext == ".json":
util.writePropertiesToConfigJSONFile(
filepath, filepath, values, ["global"],
"site.hbase-site.")
elif fext == ".properties":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".cfg":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".conf":
util.writePropertiesToConfFile(filepath, filepath, values)
elif fext == ".sh":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".yaml":
text = ""
for k, v in values.iteritems():
text += k + " : " + v
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".cmd":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext is None or fext == "" or fext == ".include":
text = ""
isFirst = True
for value in values:
if isFirst:
text += value
else:
text += "\n" + value
isFirst = False
util.writeToFile(text, filepath, isAppend=True)
logger.info("****** Copy back the configs to remote ******")
#if node!=Machine.getfqdn():
Machine.copyFromLocal(None, node, tmpConfDir, ConfDir)
Machine.chmod('777', ConfDir, recursive=True, host=node)
