def post(self, cluster_name): try: json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) project_name = request.args.get('project', "admin") masterIP = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() sshKeyPath = dict_data.get('sshKeyPath') jar_dir = dict_data.get('jarDir') base_jar_dir = os.path.basename(jar_dir) jar_name = dict_data.get('jarName') jar_class = dict_data.get('jarClass') jar_params = dict_data.get('jarParams') if not masterIP or not sshKeyPath or not jar_dir or not jar_name or not jar_params: abort(400, message="bad parameter in request body") shell.call("scp -i %s -r %s centos@%s:/home/centos" % (sshKeyPath, jar_dir, masterIP)) if base_jar_dir == 'jars': shell.call("ssh -i %s centos@%s \"sudo su - -c \'cp -R /home/centos/%s /home/hadoop && chown -R hadoop:hadoop /home/hadoop/jars\'\"" \ % (sshKeyPath, masterIP, base_jar_dir)) else: shell.call("ssh -i %s centos@%s \"sudo su - -c \'cp -R /home/centos/%s /home/hadoop && mv /home/hadoop/%s /home/hadoop/jars && chown -R hadoop:hadoop /home/hadoop/jars\'\"" \ % (sshKeyPath, masterIP, base_jar_dir, base_jar_dir)) output = shell.call("ssh -i %s centos@%s \"sudo su - -c \'%s\' hadoop\"" \ %(sshKeyPath, masterIP, submit_job_on_hadoop_cmd(jar_name, jar_class, jar_params))).strip() return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self): try: json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) project_name = request.args.get('project', "admin") master_node_template_name = dict_data.get('masterNodeTemplateName') master_node_template_plugin = dict_data.get( 'masterNodeTemplatePlugin') master_node_template_plugin_version = dict_data.get( 'masterNodeTemplatePluginVersion') master_node_template_processes = dict_data.get( 'masterNodeTemplateProcesses') master_node_template_flavor = dict_data.get( 'masterNodeTemplateFlavor') master_node_template_floating_ip_pool = dict_data.get( 'masterNodeTemplateFloatingIpPool') worker_node_template_name = dict_data.get('workerNodeTemplateName') worker_node_template_plugin = dict_data.get( 'workerNodeTemplatePlugin') worker_node_template_plugin_version = dict_data.get( 'workerNodeTemplatePluginVersion') worker_node_template_processes = dict_data.get( 'workerNodeTemplateProcesses') worker_node_template_flavor = dict_data.get( 'workerNodeTemplateFlavor') worker_node_template_floating_ip_pool = dict_data.get( 'workerNodeTemplateFloatingIpPool') cluster_template_name = dict_data.get('clusterTemplateName') cluster_worker_count = dict_data.get('clusterWorkerCount') if not master_node_template_name or not master_node_template_plugin or not master_node_template_plugin_version \ or not master_node_template_processes or not master_node_template_flavor or not master_node_template_floating_ip_pool \ or not worker_node_template_name or not worker_node_template_plugin or not worker_node_template_plugin_version \ or not worker_node_template_processes or not worker_node_template_flavor or not worker_node_template_floating_ip_pool \ or not cluster_template_name or not cluster_worker_count: abort(400, message="bad parameter in request body") shell.call(create_sahara_node_group_template_cmd(project_name, master_node_template_name, master_node_template_plugin, \ master_node_template_plugin_version, master_node_template_processes, \ master_node_template_flavor, master_node_template_floating_ip_pool)) shell.call(create_sahara_node_group_template_cmd(project_name, worker_node_template_name, worker_node_template_plugin, \ worker_node_template_plugin_version, worker_node_template_processes, \ worker_node_template_flavor, worker_node_template_floating_ip_pool)) output = shell.call(create_sahara_cluster_template_cmd(project_name, cluster_template_name, master_node_template_name, \ worker_node_template_name, cluster_worker_count)).strip() return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def get(self, cluster_name): try: retv = {} work_path = "/home/optimizer/%s" % cluster_name if not os.path.exists(work_path): os.makedirs(work_path) project_name = request.args.get('project', "admin") output = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() retv['masterIP'] = output shell.call( write_sahara_cluster_s_slavesIP_to_file_cmd( project_name, cluster_name, work_path)).strip() return output_json("%s" % retv, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self, cluster_name): try: json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) workDir = dict_data.get('workDir') scriptName = dict_data.get('scriptName', self.scriptName) sshKeyPath = dict_data.get('sshKeyPath') masterIP = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() if not workDir or not sshKeyPath or not masterIP: abort(400, message="bad parameter in request body") output = shell.call("ssh -i %s centos@%s \"cd %s; /usr/bin/bash %s slave\"" \ %(sshKeyPath, masterIP, workDir, scriptName)) return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self, cluster_name): try: ip_file = "/home/optimizer/%s/slave" % cluster_name project_name = request.args.get('project', "admin") json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) sourceDir = dict_data.get('sourceDir', self.sourceDir) scriptName = dict_data.get('scriptName', self.scriptName) sshKeyPath = dict_data.get('sshKeyPath') masterIP = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() if not sshKeyPath or not masterIP: abort(400, message="bad parameter in request body") output = shell.call("/usr/bin/bash %s/%s %s %s %s &" \ %(sourceDir, scriptName, ip_file, sshKeyPath, masterIP), \ workdir=sourceDir) return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self, cluster_name): try: project_name = request.args.get('project', "admin") json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) masterIP = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() sshKeyPath = dict_data.get('sshKeyPath') restartDataNode = dict_data.get('restartDataNode', False) restartNameNode = dict_data.get('restartNameNode', False) restartHistoryServer = dict_data.get('restartHistoryServer', False) restartNodeManager = dict_data.get('restartNodeManager', False) vcpuNum = dict_data.get('vcpuNum') memMB = dict_data.get('memMB') vcpuNumOfContainer = dict_data.get('vcpuNumOfContainer') memMBOfContainer = dict_data.get('memMBOfContainer') if not masterIP or not sshKeyPath or not vcpuNum or not memMB or not vcpuNumOfContainer or not memMBOfContainer: abort(400, message="bad parameter in request body") else: project_name = request.args.get('project', "admin") slaveIPArray = shell.call( get_sahara_cluster_s_slavesIP_cmd(project_name, cluster_name)).strip() slaveIPArrayStr = str(slaveIPArray).replace("\n", " ") restartServicesArrayStr = "resourcemanager%s%s%s%s" \ % (" nodemanager" if to_bool(restartNodeManager) else "", \ " namenode" if to_bool(restartNameNode) else "", \ " datanode" if to_bool(restartDataNode) else "", \ " historyserver" if to_bool(restartHistoryServer) else "") script_setting_str = "master_ip=%s\nslave_ip_array=(%s)\ncluster_name=%s\nrestart_services_array=(%s)\nuser=centos\nssh_key=\\\"%s\\\"\nyarn_vcpu_new_value=\\\"<value>%s<\/value>\\\"\nyarn_mem_new_value=\\\"<value>%s<\/value>\\\"\nyarn_container_vcpu_new_value=\\\"<value>%s<\/value>\\\"\nyarn_container_mem_new_value=\\\"<value>%s<\/value>\\\"" \ % (masterIP, slaveIPArrayStr, cluster_name, restartServicesArrayStr, sshKeyPath, str(vcpuNum), str(memMB), str(vcpuNumOfContainer), str(memMBOfContainer)) work_path = "/home/optimizer/%s" % cluster_name if not os.path.exists(work_path): os.makedirs(work_path) shell.call("echo \"%s\" > %s/cluster_yarn.conf" % (script_setting_str, work_path)) shell.call( "/usr/bin/cp -f /home/optimizer/scripts/cluster_yarn.sh %s" % (work_path)) output = shell.call( "/usr/bin/bash cluster_yarn.sh reconfigure", workdir=work_path) return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self, cluster_name): try: project_name = request.args.get('project', "admin") json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) project_name = request.args.get('project', "admin") size = dict_data.get('size') worker_template_name = dict_data.get('workerTemplateName') if not size: abort(400, message="bad parameter in request body") output = shell.call( scale_sahara_cluster_cmd(project_name, cluster_name, worker_template_name, size)).strip() return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self): try: json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) project_name = request.args.get('project', "admin") cluster_name = dict_data.get('clusterName') template_name = dict_data.get('template') key_pair = dict_data.get('keyPair') neutron_private_network = dict_data.get('privateNetwork') image = dict_data.get('image') if not project_name or not cluster_name or not template_name or not key_pair or not neutron_private_network or not image: abort(400, message="bad parameter in request body") output = shell.call(create_sahara_cluster_from_template_cmd(project_name, cluster_name, template_name, \ key_pair, neutron_private_network, image)).strip() return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self, cluster_name): try: project_name = request.args.get('project', "admin") json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) masterIP = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() sshKeyPath = dict_data.get('sshKeyPath') rpms_dir = dict_data.get('rpmsDir', '/home/optimizer/rpms/ganglia-rpms') if not masterIP or not sshKeyPath: abort(400, message="bad parameter in request body") else: project_name = request.args.get('project', "admin") slaveIPArray = shell.call( get_sahara_cluster_s_slavesIP_cmd(project_name, cluster_name)).strip() slaveIPArrayStr = str(slaveIPArray).replace("\n", " ") script_setting_str = "master_ip=%s\nslave_ip_array=(%s)\ncluster_name=%s\nuser=centos\nssh_key=\\\"%s\\\"\nganglia_rpms=\\\"%s\\\"\n" \ % (masterIP, slaveIPArrayStr, cluster_name, sshKeyPath, rpms_dir) work_path = "/home/optimizer/%s" % cluster_name if not os.path.exists(work_path): os.makedirs(work_path) shell.call("echo \"%s\" > %s/cluster_yarn.conf" % (script_setting_str, work_path)) shell.call( "/usr/bin/cp -f /home/optimizer/scripts/cluster_yarn.sh %s" % (work_path)) output = shell.call( "/usr/bin/bash cluster_yarn.sh restart-ganglia", workdir=work_path) return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")
def post(self, cluster_name): try: work_path = "/home/optimizer/%s" % cluster_name if not os.path.exists(work_path): os.makedirs(work_path) project_name = request.args.get('project', "admin") json_data = request.get_json(force=True) json_str = json.dumps(json_data) dict_data = JSONDecoder().decode(json_str) jobID = dict_data.get('jobID') workDir = dict_data.get('workDir') compute_node_max_cpu_core = dict_data.get('computeNodeMaxCpuCore') compute_node_max_memory_gb = dict_data.get( 'computeNodeMaxMemoryGb') compute_node_num = dict_data.get('computeNodeNum') scriptName = dict_data.get('scriptName', self.scriptName) sshKeyPath = dict_data.get('sshKeyPath') masterIP = shell.call( get_sahara_cluster_s_masterIP_cmd(project_name, cluster_name)).strip() if not workDir or not sshKeyPath or not masterIP or not jobID or not compute_node_max_cpu_core or not compute_node_max_memory_gb or not compute_node_num: abort(400, message="bad parameter in request body") jhist_json_path = "%s/%s-trace.json" % (work_path, jobID) topology_json_path = "%s/%s-topology.json" % (work_path, jobID) jhist_path = shell.call("ssh -i %s centos@%s \"sudo su - -c \'%s\' hadoop\"" \ %(sshKeyPath, masterIP, find_jhist_file_in_hdfs_cmd(jobID))).strip() shell.call("ssh -i %s centos@%s \"sudo su - -c \'%s\' hadoop\"" \ %(sshKeyPath, masterIP, analysis_job_with_hadoop_rumen_cmd(cluster_name, jobID, jhist_path))) shell.call("ssh -i %s centos@%s \"sudo su - -c \'cat /home/hadoop/%s/%s-trace.json\' hadoop\" > %s" \ % (sshKeyPath, masterIP, cluster_name, jobID, jhist_json_path)) shell.call("ssh -i %s centos@%s \"sudo su - -c \'cat /home/hadoop/%s/%s-topology.json\' hadoop\" > %s" \ % (sshKeyPath, masterIP, cluster_name, jobID, topology_json_path)) get_yarn_max_cpu = shell.call( "ssh -i %s centos@%s \"sudo grep -A 1 \'yarn.scheduler.maximum-allocation-vcores\' /opt/hadoop/etc/hadoop/yarn-site.xml | tail -1 \"" % (sshKeyPath, masterIP)).strip() get_yarn_max_memory_mb = shell.call( "ssh -i %s centos@%s \"sudo grep -A 1 \'yarn.scheduler.maximum-allocation-mb\' /opt/hadoop/etc/hadoop/yarn-site.xml | tail -1 \"" % (sshKeyPath, masterIP)).strip() get_yarn_container_cpu = shell.call( "ssh -i %s centos@%s \"sudo grep -A 1 \'yarn.scheduler.minimum-allocation-vcores\' /opt/hadoop/etc/hadoop/yarn-site.xml | tail -1 \"" % (sshKeyPath, masterIP)).strip() get_yarn_container_memory_mb = shell.call( "ssh -i %s centos@%s \"sudo grep -A 1 \'yarn.scheduler.minimum-allocation-mb\' /opt/hadoop/etc/hadoop/yarn-site.xml | tail -1 \"" % (sshKeyPath, masterIP)).strip() yarn_max_cpu = int(re.sub( r'\D', '', get_yarn_max_cpu)) if get_yarn_max_cpu else 8 yarn_max_memory_mb = int(re.sub( r'\D', '', get_yarn_max_memory_mb)) if get_yarn_max_memory_mb else 8192 yarn_container_cpu = int(re.sub( r'\D', '', get_yarn_container_cpu)) if get_yarn_container_cpu else 1 yarn_container_memory_mb = int( re.sub(r'\D', '', get_yarn_container_memory_mb) ) if get_yarn_container_memory_mb else 1024 yarn_cluster_workers_number = self._get_workers_number_from_topology_json( topology_json_path) actual_workers = int( shell.call( "ssh -i %s centos@%s \"grep %s /etc/hosts | wc -l\"" % (sshKeyPath, masterIP, cluster_name)).strip()) - 1 output = self._analysis_jhist_json( jhist_json_path, yarn_cluster_workers_number, actual_workers, yarn_max_memory_mb, yarn_max_cpu, yarn_container_memory_mb, yarn_container_cpu, compute_node_max_memory_gb, compute_node_max_cpu_core, compute_node_num) return output_json(output, 200) except Exception: log.exception(traceback.format_exc()) abort(400, message="Request failed")