def deleteConfig( # pylint: disable=redefined-builtin
changes,
confDir,
updatedConfDir,
nodes,
isFirstUpdate=True,
id=None):
backuploc = getBackupConfigLocation(id=id)
if Machine.type() == 'Windows':
# clean up the backup
Machine.rm(None, Machine.getfqdn(), backuploc, isdir=True, passwd=None)
util.copyDir(confDir, backuploc)
if isFirstUpdate:
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
if isFirstUpdate:
util.copyDir(confDir, tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file: %s", filepath)
#only supports xml file for now
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
util.deletePropertyFromXML(filepath, filepath, values)
elif fext == ".properties":
logger.info(
".properties file not supported for deleting config")
elif fext == ".sh":
logger.info(".sh format not supported for deleting config")
elif fext is None or fext == "" or fext == ".include":
logger.info(
".incluse format not supported for deleting config")
# in windows world copy the configs back to the src location
if Machine.type() == 'Windows':
for node in nodes:
for filename in changes.keys():
Machine.copyFromLocal(None,
node,
os.path.join(tmpConfDir, filename),
os.path.join(confDir, filename),
passwd=None)
else:
for node in nodes:
Machine.rm(None, node, updatedConfDir, isdir=True)
Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def modifyConfigChaosMonkey(changes,
confDir,
updatedConfDir,
nodes,
isFirstUpdate=True):
if Machine.type() == 'Windows':
# clean up the backup
Machine.rm(None,
Machine.getfqdn(),
BACKUP_CONFIG_LOCATION,
isdir=True,
passwd=None)
util.copyDir(confDir, BACKUP_CONFIG_LOCATION)
if isFirstUpdate:
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
if isFirstUpdate:
util.copyDir(confDir, tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file: %s", filepath)
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
lines = file(filepath, 'r').readlines()
del lines[-1]
file(filepath, 'w').writelines(lines)
logger.info("Modifying file: %s", filepath)
util.writeToFile(values, filepath, isAppend=True)
# in windows world copy the configs back to the src location
if Machine.type() == 'Windows':
for node in nodes:
for filename in changes.keys():
Machine.copyFromLocal(None,
node,
os.path.join(tmpConfDir, filename),
os.path.join(confDir, filename),
passwd=None)
else:
for node in nodes:
Machine.rm(None, node, updatedConfDir, isdir=True)
Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def update_lr_job_parameters(self):
from beaver import beaverConfigs
beaverConfigs.setConfigs()
# get JAVA_HOME from the hadoop configs
hadoopConfDir = Config.get('hadoop', 'HADOOP_CONF')
hadoopEnvFile = os.path.join(hadoopConfDir, 'hadoop-env.sh')
javaHome = util.getPropertyValueFromFile(hadoopEnvFile, 'JAVA_HOME')
Config.set('machine', 'JAVA_HOME', javaHome, overwrite=True)
Config.set('machine',
'JAVA_CMD',
os.path.join(javaHome, 'bin', 'java'),
overwrite=True)
AMBARI_PROP_FILE = os.path.join(Config.getEnv('WORKSPACE'), '..',
'ambari_deploy', 'uifrm',
'ambari.properties')
self.read_and_update_job_parameters()
#shutil.copy(AMBARI_PROP_FILE, self.LOCAL_WORK_DIR)
Config.set('xasecure', 'XA_KMS_HOME', '')
def modifyConfig( # pylint: disable=redefined-builtin
changes,
confDir,
updatedConfDir,
nodes,
isFirstUpdate=True,
makeCurrConfBackupInWindows=True,
id=None):
'''
Modifies hadoop config or config with similar structure.
Returns None.
Linux:
1. Create tmpModifyConfDir_<time> in artifacts dir based on source config directory in gateway
2. Modify contents in created directory.
3. Copy the directory to /tmp/hadoopConf in target machines
Windows:
1. If makeCurrConfBackupInWindows is True, backup current config first.
Copy current config to artifacts/HDPStackBackupConfig
2. Create tmpModifyConfDir_<time> in gateway.
3. Modify contents in created directory.
4. Copy the directory to target machines. Replace config in default locations in remote machines.
Calling modifyConfig twice, changes will be cumulative.
'''
backuploc = getBackupConfigLocation(id=id)
if Machine.type() == 'Windows' and makeCurrConfBackupInWindows:
# clean up the backup
Machine.rm(None, Machine.getfqdn(), backuploc, isdir=True, passwd=None)
util.copyReadableFilesFromDir(confDir, backuploc)
if isFirstUpdate:
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
if isFirstUpdate:
util.copyReadableFilesFromDir(confDir, tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file: %s", filepath)
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
util.writePropertiesToConfigXMLFile(filepath, filepath, values)
elif fext == ".json":
util.writePropertiesToConfigJSONFile(filepath, filepath,
values, ["global"],
"site.hbase-site.")
elif fext == ".properties":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".cfg":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".conf":
util.writePropertiesToConfFile(filepath, filepath, values)
elif fext == ".ini":
# 'shiro.ini : {'section:prop' : 'val}
util.writePropertiesToIniFile(filepath, filepath, values)
elif fext == ".sh":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".yaml":
text = ""
for k, v in values.iteritems():
text += k + " : " + v
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".cmd":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext is None or fext == "" or fext == ".include":
text = ""
isFirst = True
for value in values:
if isFirst:
text += value
else:
text += "\n" + value
isFirst = False
util.writeToFile(text, filepath, isAppend=True)
# in windows world copy the configs back to the src location
if Machine.type() == 'Windows':
for node in nodes:
for filename in changes.keys():
Machine.copyFromLocal(None,
node,
os.path.join(tmpConfDir, filename),
os.path.join(confDir, filename),
passwd=None)
else:
for node in nodes:
Machine.rm(user=Machine.getAdminUser(),
host=node,
filepath=updatedConfDir,
isdir=True,
passwd=Machine.getAdminPasswd())
Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def modifyConfigRemote(changes, OriginalConfDir, ConfDir, nodes, id=None): # pylint: disable=redefined-builtin
'''
Modifies hadoop config or config with similar structure.
Returns None.
Linux:
1. Create tmpModifyConfDir_<time> in artifacts dir based on source config directory in gateway
2. Modify contents in created directory.
3. Copy the directory to /tmp/hadoopConf in target machines
'''
_backuploc = getBackupConfigLocation(id=id)
tmpConfDir = os.path.join(
ARTIFACTS_DIR,
'tmpModifyConfDir_' + str(int(round(time.time() * 1000))))
Config.set(PYTHON_CONFIG_NAME,
TMP_CONF_DIR_VAR,
tmpConfDir,
overwrite=True)
tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR)
for node in nodes:
Machine.rm(Machine.getAdminUser(), node, ConfDir, isdir=True)
Machine.rm(Machine.getAdminUser(),
Machine.getfqdn(),
tmpConfDir,
isdir=True)
logger.info("*** COPY ORIGINAL CONFIGS FROM REMOTE TO LOCAL ***")
Machine.copyToLocal(None, node, OriginalConfDir, tmpConfDir)
#if node == Machine.getfqdn():
# Machine.copy(OriginalConfDir,tmpConfDir)
for filename, values in changes.items():
filepath = os.path.join(tmpConfDir, filename)
if os.path.isfile(filepath):
logger.info("Modifying file locally: %s", filepath)
_fname, fext = os.path.splitext(filepath)
if fext == ".xml":
util.writePropertiesToConfigXMLFile(
filepath, filepath, values)
elif fext == ".json":
util.writePropertiesToConfigJSONFile(
filepath, filepath, values, ["global"],
"site.hbase-site.")
elif fext == ".properties":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".cfg":
util.writePropertiesToFile(filepath, filepath, values)
elif fext == ".conf":
util.writePropertiesToConfFile(filepath, filepath, values)
elif fext == ".sh":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".yaml":
text = ""
for k, v in values.iteritems():
text += k + " : " + v
util.writeToFile(text, filepath, isAppend=True)
elif fext == ".cmd":
text = ""
for value in values:
text += "\n" + value
util.writeToFile(text, filepath, isAppend=True)
elif fext is None or fext == "" or fext == ".include":
text = ""
isFirst = True
for value in values:
if isFirst:
text += value
else:
text += "\n" + value
isFirst = False
util.writeToFile(text, filepath, isAppend=True)
logger.info("****** Copy back the configs to remote ******")
#if node!=Machine.getfqdn():
Machine.copyFromLocal(None, node, tmpConfDir, ConfDir)
Machine.chmod('777', ConfDir, recursive=True, host=node)
