def updateCapacityScheduler(registry, capacitySchedulerConfigFile): print("Saving New Capacity Schduler Config") saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile) print("Refreshing Queues with the Changes") AmbariUtils.refreshQueues(registry)
def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile): logging.info("Saving New Capacity Scheduler Config"); schedulerConf = open(capacitySchedulerConfigFile, 'r') schedulerData = schedulerConf.read() schedulerConf.close() schedulerData=str.replace(schedulerData,'version1400618703537','version'+str(int(time.time()*1000))) logging.debug(schedulerData) AmbariUtils.restPUT(registry, registry['clusterName'],schedulerData)
def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile): logging.info("Saving New Capacity Scheduler Config") schedulerConf = open(capacitySchedulerConfigFile, 'r') schedulerData = schedulerConf.read() schedulerConf.close() schedulerData = str.replace(schedulerData, 'version1400618703537', 'version' + str(int(time.time() * 1000))) logging.debug(schedulerData) AmbariUtils.restPUT(registry, registry['clusterName'], schedulerData)
def updateCapacityScheduler(registry, capacitySchedulerConfigFile): print("Saving New Capacity Schduler Config") saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile) print("Refreshing Queues with the Changes") AmbariUtils.refreshQueues(registry) # registry = AmbariUtils.createRegistry("../config.properties"); # capacitySchedulerConfigFile = "../../cluster/capacity-scheduler/csConfig-desired.json" # updateCapacityScheduler(registry, capacitySchedulerConfigFile)
def updateYarnConfigForLabels(registry): # Update with new capacity scheduler config print("Updating Yarn Configuration for Node Labels.") yarnSiteVersion = getLatestYarnVersion(registry) configurePropertiesForYarnLabels(registry, yarnSiteVersion) # restart yarn to take affect print("Restarting Yarn Service to Configuration to Take Affect.") AmbariUtils.stopYarnService(registry) time.sleep(int(40)) AmbariUtils.startYarnService(registry) time.sleep(int(120))
def updateYarnConfigForLabels(registry): # Update with new capacity scheduler config print ("Updating Yarn Configuration for Node Labels.") yarnSiteVersion = getLatestYarnVersion(registry) configurePropertiesForYarnLabels(registry, yarnSiteVersion); # restart yarn to take affect print ("Restarting Yarn Service to Configuration to Take Affect.") AmbariUtils.stopYarnService(registry); time.sleep(int(40)) AmbariUtils.startYarnService(registry); time.sleep(int(120))
def configurePropertiesForYarnLabels(registry, yarnSiteVersion): #Get Current YARN-SITE & Add Preemption Values to Dict orgYARNProps = (json.loads(AmbariUtils.restGET(registry, registry['clusterName']+'/configurations?type=yarn-site&tag='+yarnSiteVersion)))['items'][0]['properties'] logging.info(orgYARNProps); orgYARNProps['yarn.node-labels.manager-class']='org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager' orgYARNProps['yarn.node-labels.fs-store.root-dir']='hdfs://vetticaden01.cloud.hortonworks.com:8020/yarn/node-labels' # On a Typical VM Node where you have less than 8 GB of memory, decrease the memory to 682 from 1024 or slider with node labels wont' work orgYARNProps['yarn.scheduler.minimum-allocation-mb']='682' #fix single quotes pythng dict issue with json.dumps orgYARNProps = json.dumps(orgYARNProps) newYARNSite = json.loads('{"Clusters": {"desired_config": {"type": "yarn-site","tag": "version'+str(int(time.time()*1000))+'", "properties": '+ orgYARNProps +'}}}') logging.info('newYARNSite ' + json.dumps(newYARNSite)) AmbariUtils.restPUT(registry, registry['clusterName'], json.dumps(newYARNSite))
def configurePropertiesForYarnLabels(registry, yarnSiteVersion): #Get Current YARN-SITE & Add Preemption Values to Dict orgYARNProps = (json.loads( AmbariUtils.restGET( registry, registry['clusterName'] + '/configurations?type=yarn-site&tag=' + yarnSiteVersion)))['items'][0]['properties'] logging.info(orgYARNProps) orgYARNProps[ 'yarn.node-labels.manager-class'] = 'org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager' orgYARNProps[ 'yarn.node-labels.fs-store.root-dir'] = 'hdfs://vetticaden01.cloud.hortonworks.com:8020/yarn/node-labels' # On a Typical VM Node where you have less than 8 GB of memory, decrease the memory to 682 from 1024 or slider with node labels wont' work orgYARNProps['yarn.scheduler.minimum-allocation-mb'] = '682' #fix single quotes pythng dict issue with json.dumps orgYARNProps = json.dumps(orgYARNProps) newYARNSite = json.loads( '{"Clusters": {"desired_config": {"type": "yarn-site","tag": "version' + str(int(time.time() * 1000)) + '", "properties": ' + orgYARNProps + '}}}') logging.info('newYARNSite ' + json.dumps(newYARNSite)) AmbariUtils.restPUT(registry, registry['clusterName'], json.dumps(newYARNSite))
def masterSetup(): registry = AmbariUtils.createRegistry("config.properties"); #do initialization required for labels subprocess.call(["labels/nodeLabelsSetup.sh"], shell=True) updateYarnConfigForLabels(registry) # Now Create the node labels and assign it to nodes subprocess.call(["labels/nodeLabelCreationAndAssignment.sh"], shell=True) #update capacity scheduler configs and refresh queues capacitySchedulerConfigFile = "../cluster/capacity-scheduler/csConfig-desired.json" updateCapacityScheduler(registry, capacitySchedulerConfigFile);
0, "[REPLACE]/hdp/reference-apps/iot-trucking-app/trucking-env-setup/environment/prod" ) from setup.common import AmbariUtils def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile): logging.info("Saving New Capacity Scheduler Config") schedulerConf = open(capacitySchedulerConfigFile, 'r') schedulerData = schedulerConf.read() schedulerConf.close() schedulerData = str.replace(schedulerData, 'version1400618703537', 'version' + str(int(time.time() * 1000))) logging.debug(schedulerData) AmbariUtils.restPUT(registry, registry['clusterName'], schedulerData) def updateCapacityScheduler(registry, capacitySchedulerConfigFile): print("Saving New Capacity Schduler Config") saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile) print("Refreshing Queues with the Changes") AmbariUtils.refreshQueues(registry) registry = AmbariUtils.createRegistry("../config.properties") capacitySchedulerConfigFile = "../../cluster/capacity-scheduler/csConfig-desired.json" updateCapacityScheduler(registry, capacitySchedulerConfigFile)
# Replace the below to ensure to right the path sys.path.insert(0, "[REPLACE]/hdp/reference-apps/iot-trucking-app/trucking-env-setup/environment/prod") from setup.common import AmbariUtils def saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile): logging.info("Saving New Capacity Scheduler Config"); schedulerConf = open(capacitySchedulerConfigFile, 'r') schedulerData = schedulerConf.read() schedulerConf.close() schedulerData=str.replace(schedulerData,'version1400618703537','version'+str(int(time.time()*1000))) logging.debug(schedulerData) AmbariUtils.restPUT(registry, registry['clusterName'],schedulerData) def updateCapacityScheduler(registry, capacitySchedulerConfigFile): print("Saving New Capacity Schduler Config") saveNewCapacitySchedulerConfig(registry, capacitySchedulerConfigFile) print("Refreshing Queues with the Changes") AmbariUtils.refreshQueues(registry) registry = AmbariUtils.createRegistry("../config.properties"); capacitySchedulerConfigFile = "../../cluster/capacity-scheduler/csConfig-desired.json" updateCapacityScheduler(registry, capacitySchedulerConfigFile)
def getLatestYarnVersion(registry): currentConfigs = AmbariUtils.restGET( registry, registry['clusterName'] + '/configurations') yarnSiteVersion = AmbariUtils.vectVersion(currentConfigs, 'yarn-site') return yarnSiteVersion
def getLatestYarnVersion(registry): currentConfigs = AmbariUtils.restGET(registry, registry['clusterName']+'/configurations'); yarnSiteVersion = AmbariUtils.vectVersion(currentConfigs, 'yarn-site') return yarnSiteVersion