def updateCapacityScheduler(registry, capacitySchedulerConfigFile):
print("Saving New Capacity Schduler Config")
saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile)
print("Refreshing Queues with the Changes")
AmbariUtils.refreshQueues(registry)
def updateCapacityScheduler(registry, capacitySchedulerConfigFile):
print("Saving New Capacity Schduler Config")
saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile)
print("Refreshing Queues with the Changes")
AmbariUtils.refreshQueues(registry)
def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile):
logging.info("Saving New Capacity Scheduler Config");
schedulerConf = open(capacitySchedulerConfigFile, 'r')
schedulerData = schedulerConf.read()
schedulerConf.close()
schedulerData=str.replace(schedulerData,'version1400618703537','version'+str(int(time.time()*1000)))
logging.debug(schedulerData)
AmbariUtils.restPUT(registry, registry['clusterName'],schedulerData)
def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile):
logging.info("Saving New Capacity Scheduler Config")
schedulerConf = open(capacitySchedulerConfigFile, 'r')
schedulerData = schedulerConf.read()
schedulerConf.close()
schedulerData = str.replace(schedulerData, 'version1400618703537',
'version' + str(int(time.time() * 1000)))
logging.debug(schedulerData)
AmbariUtils.restPUT(registry, registry['clusterName'], schedulerData)
def updateCapacityScheduler(registry, capacitySchedulerConfigFile):
print("Saving New Capacity Schduler Config")
saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile)
print("Refreshing Queues with the Changes")
AmbariUtils.refreshQueues(registry)
# registry = AmbariUtils.createRegistry("../config.properties");
# capacitySchedulerConfigFile = "../../cluster/capacity-scheduler/csConfig-desired.json"
# updateCapacityScheduler(registry, capacitySchedulerConfigFile)
def updateYarnConfigForLabels(registry):
# Update with new capacity scheduler config
print("Updating Yarn Configuration for Node Labels.")
yarnSiteVersion = getLatestYarnVersion(registry)
configurePropertiesForYarnLabels(registry, yarnSiteVersion)
# restart yarn to take affect
print("Restarting Yarn Service to Configuration to Take Affect.")
AmbariUtils.stopYarnService(registry)
time.sleep(int(40))
AmbariUtils.startYarnService(registry)
time.sleep(int(120))
def updateYarnConfigForLabels(registry):
# Update with new capacity scheduler config
print ("Updating Yarn Configuration for Node Labels.")
yarnSiteVersion = getLatestYarnVersion(registry)
configurePropertiesForYarnLabels(registry, yarnSiteVersion);
# restart yarn to take affect
print ("Restarting Yarn Service to Configuration to Take Affect.")
AmbariUtils.stopYarnService(registry);
time.sleep(int(40))
AmbariUtils.startYarnService(registry);
time.sleep(int(120))
def configurePropertiesForYarnLabels(registry, yarnSiteVersion):
#Get Current YARN-SITE & Add Preemption Values to Dict
orgYARNProps = (json.loads(AmbariUtils.restGET(registry, registry['clusterName']+'/configurations?type=yarn-site&tag='+yarnSiteVersion)))['items'][0]['properties']
logging.info(orgYARNProps);
orgYARNProps['yarn.node-labels.manager-class']='org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager'
orgYARNProps['yarn.node-labels.fs-store.root-dir']='hdfs://vetticaden01.cloud.hortonworks.com:8020/yarn/node-labels'
# On a Typical VM Node where you have less than 8 GB of memory, decrease the memory to 682 from 1024 or slider with node labels wont' work
orgYARNProps['yarn.scheduler.minimum-allocation-mb']='682'
#fix single quotes pythng dict issue with json.dumps
orgYARNProps = json.dumps(orgYARNProps)
newYARNSite = json.loads('{"Clusters": {"desired_config": {"type": "yarn-site","tag": "version'+str(int(time.time()*1000))+'", "properties": '+ orgYARNProps +'}}}')
logging.info('newYARNSite ' + json.dumps(newYARNSite))
AmbariUtils.restPUT(registry, registry['clusterName'], json.dumps(newYARNSite))
def configurePropertiesForYarnLabels(registry, yarnSiteVersion):
#Get Current YARN-SITE & Add Preemption Values to Dict
orgYARNProps = (json.loads(
AmbariUtils.restGET(
registry,
registry['clusterName'] + '/configurations?type=yarn-site&tag=' +
yarnSiteVersion)))['items'][0]['properties']
logging.info(orgYARNProps)
orgYARNProps[
'yarn.node-labels.manager-class'] = 'org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager'
orgYARNProps[
'yarn.node-labels.fs-store.root-dir'] = 'hdfs://vetticaden01.cloud.hortonworks.com:8020/yarn/node-labels'
# On a Typical VM Node where you have less than 8 GB of memory, decrease the memory to 682 from 1024 or slider with node labels wont' work
orgYARNProps['yarn.scheduler.minimum-allocation-mb'] = '682'
#fix single quotes pythng dict issue with json.dumps
orgYARNProps = json.dumps(orgYARNProps)
newYARNSite = json.loads(
'{"Clusters": {"desired_config": {"type": "yarn-site","tag": "version'
+ str(int(time.time() * 1000)) + '", "properties": ' + orgYARNProps +
'}}}')
logging.info('newYARNSite ' + json.dumps(newYARNSite))
AmbariUtils.restPUT(registry, registry['clusterName'],
json.dumps(newYARNSite))
def masterSetup():
registry = AmbariUtils.createRegistry("config.properties");
#do initialization required for labels
subprocess.call(["labels/nodeLabelsSetup.sh"], shell=True)
updateYarnConfigForLabels(registry)
# Now Create the node labels and assign it to nodes
subprocess.call(["labels/nodeLabelCreationAndAssignment.sh"], shell=True)
#update capacity scheduler configs and refresh queues
capacitySchedulerConfigFile = "../cluster/capacity-scheduler/csConfig-desired.json"
updateCapacityScheduler(registry, capacitySchedulerConfigFile);
0,
"[REPLACE]/hdp/reference-apps/iot-trucking-app/trucking-env-setup/environment/prod"
)
from setup.common import AmbariUtils
def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile):
logging.info("Saving New Capacity Scheduler Config")
schedulerConf = open(capacitySchedulerConfigFile, 'r')
schedulerData = schedulerConf.read()
schedulerConf.close()
schedulerData = str.replace(schedulerData, 'version1400618703537',
'version' + str(int(time.time() * 1000)))
logging.debug(schedulerData)
AmbariUtils.restPUT(registry, registry['clusterName'], schedulerData)
def updateCapacityScheduler(registry, capacitySchedulerConfigFile):
print("Saving New Capacity Schduler Config")
saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile)
print("Refreshing Queues with the Changes")
AmbariUtils.refreshQueues(registry)
registry = AmbariUtils.createRegistry("../config.properties")
capacitySchedulerConfigFile = "../../cluster/capacity-scheduler/csConfig-desired.json"
updateCapacityScheduler(registry, capacitySchedulerConfigFile)
# Replace the below to ensure to right the path
sys.path.insert(0, "[REPLACE]/hdp/reference-apps/iot-trucking-app/trucking-env-setup/environment/prod")
from setup.common import AmbariUtils
def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile):
logging.info("Saving New Capacity Scheduler Config");
schedulerConf = open(capacitySchedulerConfigFile, 'r')
schedulerData = schedulerConf.read()
schedulerConf.close()
schedulerData=str.replace(schedulerData,'version1400618703537','version'+str(int(time.time()*1000)))
logging.debug(schedulerData)
AmbariUtils.restPUT(registry, registry['clusterName'],schedulerData)
def updateCapacityScheduler(registry, capacitySchedulerConfigFile):
print("Saving New Capacity Schduler Config")
saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile)
print("Refreshing Queues with the Changes")
AmbariUtils.refreshQueues(registry)
registry = AmbariUtils.createRegistry("../config.properties");
capacitySchedulerConfigFile = "../../cluster/capacity-scheduler/csConfig-desired.json"
updateCapacityScheduler(registry, capacitySchedulerConfigFile)
def getLatestYarnVersion(registry):
currentConfigs = AmbariUtils.restGET(
registry, registry['clusterName'] + '/configurations')
yarnSiteVersion = AmbariUtils.vectVersion(currentConfigs, 'yarn-site')
return yarnSiteVersion
def getLatestYarnVersion(registry):
currentConfigs = AmbariUtils.restGET(registry, registry['clusterName']+'/configurations');
yarnSiteVersion = AmbariUtils.vectVersion(currentConfigs, 'yarn-site')
return yarnSiteVersion
