def get_job_log_by_jobname_special_k8s(_jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) if (_param.has_key("job_common_submit_type")): job_common_submit_type = _param["job_common_submit_type"] if (_param.has_key("job_common_is_distributed")): job_common_is_distributed = _param["job_common_is_distributed"] if (_param.has_key("cluster_dispatch_type")): cluster_dispatch_type = _param["cluster_dispatch_type"] if (_param.has_key("job_query_command")): # job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po""" job_query_command = "sh %s %s" % (_param["job_log_command"], _param["query_name"]) if (job_common_submit_type == "tensorflow" and job_common_is_distributed == "1" and cluster_dispatch_type == "k8s"): query_status_param = {} query_status_param["job_name"] = _jobname query_status_param["command"] = job_query_command # url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname) host_url = gen_host(_param) + "/queryJobLog.do" job_info = post(host_url, query_status_param) else: url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) logger.debug(job_info) print "JOB LOG LIST FOLLOW : " print job_info
def client_kill_job(_job_name="Empty", _job_stop_command="Empty", _param_map={}): logger = wei_logger("wei_kill_job") logger.info("Job Stop Command: %s" % _job_stop_command) logger.info("Job Name: %s" % _job_name) # if(_param_map.has_key("client_change_host") and # _param_map["client_change_host"] == "true"): # client_center_host_name = get_or_else(_param_map, # "client_center_host_name", # "http://controlcenter.ds.sina.com.cn") # if(_param_map.has_key("client_control_center_version")): # client_control_center_version = get_or_else(_param_map, # "client_control_center_version", # "controlCenter-1.0.0") client_center_host_name = url_helper.gen_host(_param_map) if (int(_param_map.get("job_common_is_docker")) == 0): url = url_helper.gen_url_kill_job( _client_center_host_name=client_center_host_name, _job_name=_job_name, _job_stop_command=_job_stop_command) post_interface = client_center_host_name + "/killJob.do" req_info = http_helper.post(post_interface, _param_map) elif (int(_param_map.get("job_common_is_docker")) == 1): post_interface = client_center_host_name + "/killJob.do" req_info = http_helper.post(post_interface, _param_map) else: logger.error("Job config error: 'Job_common_is_docker'") exit(1) kill_request_info_check(req_info)
def get_group_info_by_username(_username, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/account_group.do?account=" % host_name + _username response = urllib2.urlopen(url) try: cluster_info = json.loads(response.read())[0] except (IndexError, StandardError), e: logger.error("User Name %s has Wrong!!" % _username)
def get_job_info_by_jobname(_jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) show_info = format_job_info(job_info) print "JOB INFO LIST FOLLOW : " for show_info_line in show_info: print show_info_line
def get_all_job(_job_name="", _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = '%s/query.do?op=query&query_type=job&job_name=%s' % (host_name, _job_name) job_info = get(url) show_info = format_job_info(job_info) print "JOB INFO LIST FOLLOW : " if (type(show_info) == str): for show_info_line in show_info: print show_info_line else: print show_info
def get_job_by_username_with_jobname(_username, _jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) result = json.loads(job_info).get("result") for job_elem in result: show_info = "" for key in filter( lambda _key: (_key.find("jobName") >= 0 or _key.find( "status") >= 0 or _key.find("jobUrl") >= 0 or _key.find( "submitTime") >= 0 or _key.find("clusterType") >= 0), job_elem): if job_elem['account'] == _username: show_info += "%s : %s\n\t" % (key, job_elem[key]) logger.info("\n" + show_info)
def get_all_cluster(_cluster_name="", _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/query.do?op=query&queryType=cluster&clusterName=%s" % ( host_name, _cluster_name) response = urllib2.urlopen(url) cluster_info = response.read() result = json.loads(cluster_info).get("result") for cluster_elem in result: show_info = "" for key in filter( lambda _key: (_key.find("version") >= 0 or _key.find("clusterName") >= 0 or _key .find("uiUrl") >= 0 or _key.find("clusterType") >= 0), cluster_elem): show_info += "%s : %s\n\t" % (key, cluster_elem[key]) logger.info("\n" + show_info)
def get_job_info_by_jobname_special_k8s(_jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) if (_param.has_key("job_common_submit_type")): job_common_submit_type = _param["job_common_submit_type"] if (_param.has_key("job_common_is_distributed")): job_common_is_distributed = _param["job_common_is_distributed"] if (_param.has_key("cluster_dispatch_type")): cluster_dispatch_type = _param["cluster_dispatch_type"] if (_param.has_key("job_query_command")): # job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po""" job_query_command = "sh %s po" % _param["job_query_command"] if (_param.has_key("job_common_is_docker")): job_common_is_docker = _param.get("job_common_is_docker") if (job_common_submit_type == "tensorflow" and job_common_is_distributed == "1" and cluster_dispatch_type == "k8s"): query_status_param = {} query_status_param["job_name"] = _jobname query_status_param["command"] = job_query_command query_status_param["job_common_is_docker"] = job_common_is_docker # url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname) host_url = gen_host(_param) + "/queryJobStatus.do" job_info = post(host_url, query_status_param) elif (job_common_submit_type == "spark" and job_common_is_distributed == "1" and cluster_dispatch_type == "yarn"): query_status_param = {} query_status_param["job_name"] = _jobname query_status_param["command"] = "sh " + _param["job_query_command"] query_status_param["job_common_is_docker"] = job_common_is_docker # url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname) host_url = gen_host(_param) + "/queryJobStatus.do" job_info = post(host_url, query_status_param) result_info = json.loads(job_info).get("result") if (len(result_info) <= 0): logger.error("no result info received. PLEASE CHECK JOB_NAME. ") print("no result info received. PLEASE CHECK JOB_NAME. ") exit(1) show_info = result_info[0] split_show_info = show_info.split("proxy") if (len(split_show_info) > 1): EMR_MASTER = "http://10.87.49.221:8088/cluster/" APP_INFO = "app/" + split_show_info[-1] show_info = EMR_MASTER + APP_INFO logger.debug("SHOW_INFO : " + show_info) print u"TASK信息 : \n\t%s" % show_info print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=0&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], _jobname, job_common_is_docker) print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=1&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], _jobname, job_common_is_docker) exit(0) else: url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) logger.debug(job_info) show_info = format_job_info(job_info) print "JOB INFO LIST FOLLOW : " for show_info_line in show_info: print u"TASK信息 : \n\t%s" % show_info_line print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+0+%s&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], show_info_line.split(" ")[0], _jobname, job_common_is_docker) print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+1+%s&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], show_info_line.split(" ")[0], _jobname, job_common_is_docker)