def start(cls, host, hcat_confdir=None): if Machine.type() == 'Windows': if not hcat_confdir == None: HCAT_CONF_DIR = Config.get('hcatalog', 'HCATALOG_CONF') Machine.copyFromLocal(Machine.getAdminUser(), host, os.path.join(hcat_confdir, 'webhcat-site.xml'), HCAT_CONF_DIR, passwd=Machine.getAdminPasswd()) Machine.service("templeton", "start", host=host) else: env = {} if not hcat_confdir == None: env['WEBHCAT_CONF_DIR'] = hcat_confdir hcat_user = Config.get('hcatalog', 'HCATALOG_USER', 'hcat') hcat_bin = os.path.join( Config.get('hcatalog', 'HCATALOG_HOME', '/usr/hdp/current/hive-webhcat'), 'sbin') start_cmd = os.path.join(hcat_bin, 'webhcat_server.sh') + ' start' exit_code, stdout = Machine.runas(hcat_user, start_cmd, host, env=env, passwd=Machine.getAdminPasswd()) assert exit_code == 0, "Unable to start webhcat_server.sh on %s \n %s " % ( host, start_cmd)
def deleteConfig( # pylint: disable=redefined-builtin changes, confDir, updatedConfDir, nodes, isFirstUpdate=True, id=None): backuploc = getBackupConfigLocation(id=id) if Machine.type() == 'Windows': # clean up the backup Machine.rm(None, Machine.getfqdn(), backuploc, isdir=True, passwd=None) util.copyDir(confDir, backuploc) if isFirstUpdate: tmpConfDir = os.path.join( ARTIFACTS_DIR, 'tmpModifyConfDir_' + str(int(round(time.time() * 1000)))) Config.set(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR, tmpConfDir, overwrite=True) tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR) if isFirstUpdate: util.copyDir(confDir, tmpConfDir) for filename, values in changes.items(): filepath = os.path.join(tmpConfDir, filename) if os.path.isfile(filepath): logger.info("Modifying file: %s", filepath) #only supports xml file for now _fname, fext = os.path.splitext(filepath) if fext == ".xml": util.deletePropertyFromXML(filepath, filepath, values) elif fext == ".properties": logger.info( ".properties file not supported for deleting config") elif fext == ".sh": logger.info(".sh format not supported for deleting config") elif fext is None or fext == "" or fext == ".include": logger.info( ".incluse format not supported for deleting config") # in windows world copy the configs back to the src location if Machine.type() == 'Windows': for node in nodes: for filename in changes.keys(): Machine.copyFromLocal(None, node, os.path.join(tmpConfDir, filename), os.path.join(confDir, filename), passwd=None) else: for node in nodes: Machine.rm(None, node, updatedConfDir, isdir=True) Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def modifyConfigChaosMonkey(changes, confDir, updatedConfDir, nodes, isFirstUpdate=True): if Machine.type() == 'Windows': # clean up the backup Machine.rm(None, Machine.getfqdn(), BACKUP_CONFIG_LOCATION, isdir=True, passwd=None) util.copyDir(confDir, BACKUP_CONFIG_LOCATION) if isFirstUpdate: tmpConfDir = os.path.join( ARTIFACTS_DIR, 'tmpModifyConfDir_' + str(int(round(time.time() * 1000)))) Config.set(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR, tmpConfDir, overwrite=True) tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR) if isFirstUpdate: util.copyDir(confDir, tmpConfDir) for filename, values in changes.items(): filepath = os.path.join(tmpConfDir, filename) if os.path.isfile(filepath): logger.info("Modifying file: %s", filepath) _fname, fext = os.path.splitext(filepath) if fext == ".xml": lines = file(filepath, 'r').readlines() del lines[-1] file(filepath, 'w').writelines(lines) logger.info("Modifying file: %s", filepath) util.writeToFile(values, filepath, isAppend=True) # in windows world copy the configs back to the src location if Machine.type() == 'Windows': for node in nodes: for filename in changes.keys(): Machine.copyFromLocal(None, node, os.path.join(tmpConfDir, filename), os.path.join(confDir, filename), passwd=None) else: for node in nodes: Machine.rm(None, node, updatedConfDir, isdir=True) Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def restoreConfig(changes, restoreLocation, nodes, id=None): # pylint: disable=redefined-builtin ''' Restores hadoop config or config with similar structure. Return None. Linux: do nothing because default config is not altered. Windows: copy config from BACKUP_CONFIG_LOCATION back to default config directories in target machines. ''' backuploc = getBackupConfigLocation(id=id) if Machine.type() != 'Windows': return for node in nodes: for filename in changes: Machine.copyFromLocal(None, node, os.path.join(backuploc, filename), os.path.join(restoreLocation, filename), passwd=None)
def runas(cls, user, cmd, host="", cwd=None, env=None, logoutput=True, passwd=None): if Machine.isLinux(): if user is None and passwd is None: user = Machine.getAdminUser() keypairLocation = "/home/hrt_qa/.ssh/id_rsa_log_server_hrt_qa" if Machine.isHumboldt(): # for secure cluster in humboldt path is different if not os.path.exists(keypairLocation): keypairLocation = "/home/HDINSIGHT/hrt_qa/.ssh/id_rsa_log_server_hrt_qa" # pylint: disable=line-too-long cmd = "rsync -e 'ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s -o PreferredAuthentications=publickey' %s" % ( keypairLocation, ssh_port_number, cmd) # pylint: enable=line-too-long else: # pylint: disable=line-too-long cmd = "rsync -e \"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s -o PreferredAuthentications=publickey\" %s" % ( ssh_port_number, cmd) # pylint: enable=line-too-long if logoutput: logger.info("RSync.runas cmd=%s", cmd) return Machine.runas(user, cmd, host, cwd, env, logoutput, passwd, retry_count, retry_sleep_time) else: rsyncHome = "/cygdrive/c/testtools/cwrsync" rsyncLocation = "c:\\testtools\\cwrsync\\rsync.exe" #must use c:\\ so remote powershell can pick this up. keypairLocation = "c:\\testtools\\id_rsa_log_server_hrt_qa" # pylint: disable=line-too-long cmd = "%s -e \"%s/ssh -i %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p %s -o PreferredAuthentications=publickey\" %s" % \ (rsyncLocation, rsyncHome, keypairLocation, ssh_port_number, cmd) # pylint: enable=line-too-long if logoutput: logger.info("RSync.runas cmd=%s", cmd) if Machine.isSameHost(host, None): pass else: # remote # I cant find a way to call this on the fly. # The only way it can work is to create bat file for the command and run the file. batchFilename = 'rsync-%s.bat' % str(int(time.time())) tmpLocalFile = os.path.join(Config.getEnv('ARTIFACTS_DIR'), batchFilename) logger.info("write to %s. contents=%s", tmpLocalFile, cmd) util.writeToFile(cmd, tmpLocalFile) destPath = os.path.join(Machine.getTempDir(), batchFilename) Machine.copyFromLocal(Machine.getAdminUser(), host, tmpLocalFile, destPath, Machine.getAdminPasswd()) cmd = destPath return Machine.runas(Machine.getAdminUser(), cmd, host, cwd, env, logoutput, passwd=Machine.getAdminPasswd(), retry_count=retry_count, retry_sleep_time=retry_sleep_time)
def modifyConfig( # pylint: disable=redefined-builtin changes, confDir, updatedConfDir, nodes, isFirstUpdate=True, makeCurrConfBackupInWindows=True, id=None): ''' Modifies hadoop config or config with similar structure. Returns None. Linux: 1. Create tmpModifyConfDir_<time> in artifacts dir based on source config directory in gateway 2. Modify contents in created directory. 3. Copy the directory to /tmp/hadoopConf in target machines Windows: 1. If makeCurrConfBackupInWindows is True, backup current config first. Copy current config to artifacts/HDPStackBackupConfig 2. Create tmpModifyConfDir_<time> in gateway. 3. Modify contents in created directory. 4. Copy the directory to target machines. Replace config in default locations in remote machines. Calling modifyConfig twice, changes will be cumulative. ''' backuploc = getBackupConfigLocation(id=id) if Machine.type() == 'Windows' and makeCurrConfBackupInWindows: # clean up the backup Machine.rm(None, Machine.getfqdn(), backuploc, isdir=True, passwd=None) util.copyReadableFilesFromDir(confDir, backuploc) if isFirstUpdate: tmpConfDir = os.path.join( ARTIFACTS_DIR, 'tmpModifyConfDir_' + str(int(round(time.time() * 1000)))) Config.set(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR, tmpConfDir, overwrite=True) tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR) if isFirstUpdate: util.copyReadableFilesFromDir(confDir, tmpConfDir) for filename, values in changes.items(): filepath = os.path.join(tmpConfDir, filename) if os.path.isfile(filepath): logger.info("Modifying file: %s", filepath) _fname, fext = os.path.splitext(filepath) if fext == ".xml": util.writePropertiesToConfigXMLFile(filepath, filepath, values) elif fext == ".json": util.writePropertiesToConfigJSONFile(filepath, filepath, values, ["global"], "site.hbase-site.") elif fext == ".properties": util.writePropertiesToFile(filepath, filepath, values) elif fext == ".cfg": util.writePropertiesToFile(filepath, filepath, values) elif fext == ".conf": util.writePropertiesToConfFile(filepath, filepath, values) elif fext == ".ini": # 'shiro.ini : {'section:prop' : 'val} util.writePropertiesToIniFile(filepath, filepath, values) elif fext == ".sh": text = "" for value in values: text += "\n" + value util.writeToFile(text, filepath, isAppend=True) elif fext == ".yaml": text = "" for k, v in values.iteritems(): text += k + " : " + v util.writeToFile(text, filepath, isAppend=True) elif fext == ".cmd": text = "" for value in values: text += "\n" + value util.writeToFile(text, filepath, isAppend=True) elif fext is None or fext == "" or fext == ".include": text = "" isFirst = True for value in values: if isFirst: text += value else: text += "\n" + value isFirst = False util.writeToFile(text, filepath, isAppend=True) # in windows world copy the configs back to the src location if Machine.type() == 'Windows': for node in nodes: for filename in changes.keys(): Machine.copyFromLocal(None, node, os.path.join(tmpConfDir, filename), os.path.join(confDir, filename), passwd=None) else: for node in nodes: Machine.rm(user=Machine.getAdminUser(), host=node, filepath=updatedConfDir, isdir=True, passwd=Machine.getAdminPasswd()) Machine.copyFromLocal(None, node, tmpConfDir, updatedConfDir)
def modifyConfigRemote(changes, OriginalConfDir, ConfDir, nodes, id=None): # pylint: disable=redefined-builtin ''' Modifies hadoop config or config with similar structure. Returns None. Linux: 1. Create tmpModifyConfDir_<time> in artifacts dir based on source config directory in gateway 2. Modify contents in created directory. 3. Copy the directory to /tmp/hadoopConf in target machines ''' _backuploc = getBackupConfigLocation(id=id) tmpConfDir = os.path.join( ARTIFACTS_DIR, 'tmpModifyConfDir_' + str(int(round(time.time() * 1000)))) Config.set(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR, tmpConfDir, overwrite=True) tmpConfDir = Config.get(PYTHON_CONFIG_NAME, TMP_CONF_DIR_VAR) for node in nodes: Machine.rm(Machine.getAdminUser(), node, ConfDir, isdir=True) Machine.rm(Machine.getAdminUser(), Machine.getfqdn(), tmpConfDir, isdir=True) logger.info("*** COPY ORIGINAL CONFIGS FROM REMOTE TO LOCAL ***") Machine.copyToLocal(None, node, OriginalConfDir, tmpConfDir) #if node == Machine.getfqdn(): # Machine.copy(OriginalConfDir,tmpConfDir) for filename, values in changes.items(): filepath = os.path.join(tmpConfDir, filename) if os.path.isfile(filepath): logger.info("Modifying file locally: %s", filepath) _fname, fext = os.path.splitext(filepath) if fext == ".xml": util.writePropertiesToConfigXMLFile( filepath, filepath, values) elif fext == ".json": util.writePropertiesToConfigJSONFile( filepath, filepath, values, ["global"], "site.hbase-site.") elif fext == ".properties": util.writePropertiesToFile(filepath, filepath, values) elif fext == ".cfg": util.writePropertiesToFile(filepath, filepath, values) elif fext == ".conf": util.writePropertiesToConfFile(filepath, filepath, values) elif fext == ".sh": text = "" for value in values: text += "\n" + value util.writeToFile(text, filepath, isAppend=True) elif fext == ".yaml": text = "" for k, v in values.iteritems(): text += k + " : " + v util.writeToFile(text, filepath, isAppend=True) elif fext == ".cmd": text = "" for value in values: text += "\n" + value util.writeToFile(text, filepath, isAppend=True) elif fext is None or fext == "" or fext == ".include": text = "" isFirst = True for value in values: if isFirst: text += value else: text += "\n" + value isFirst = False util.writeToFile(text, filepath, isAppend=True) logger.info("****** Copy back the configs to remote ******") #if node!=Machine.getfqdn(): Machine.copyFromLocal(None, node, tmpConfDir, ConfDir) Machine.chmod('777', ConfDir, recursive=True, host=node)