def get_job_log_by_jobname_special_k8s(_jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) if (_param.has_key("job_common_submit_type")): job_common_submit_type = _param["job_common_submit_type"] if (_param.has_key("job_common_is_distributed")): job_common_is_distributed = _param["job_common_is_distributed"] if (_param.has_key("cluster_dispatch_type")): cluster_dispatch_type = _param["cluster_dispatch_type"] if (_param.has_key("job_query_command")): # job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po""" job_query_command = "sh %s %s" % (_param["job_log_command"], _param["query_name"]) if (job_common_submit_type == "tensorflow" and job_common_is_distributed == "1" and cluster_dispatch_type == "k8s"): query_status_param = {} query_status_param["job_name"] = _jobname query_status_param["command"] = job_query_command # url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname) host_url = gen_host(_param) + "/queryJobLog.do" job_info = post(host_url, query_status_param) else: url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) logger.debug(job_info) print "JOB LOG LIST FOLLOW : " print job_info
def check_update(_version="0.5.2.0", _size=""): url = "http://10.77.29.69:8080/controlCenter-1.0.0/api/clientUpdate" url = "http://10.77.29.69:8080/waic/api/clientUpdate" req_dict = json.loads(get(url)) new_result = req_dict.get("result")[0] new_version = new_result.get("version") new_size = new_result.get("size") return not (_version == new_version and _size == new_size)
def get_job_info_by_jobname(_jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) show_info = format_job_info(job_info) print "JOB INFO LIST FOLLOW : " for show_info_line in show_info: print show_info_line
def get_all_job(_job_name="", _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = '%s/query.do?op=query&query_type=job&job_name=%s' % (host_name, _job_name) job_info = get(url) show_info = format_job_info(job_info) print "JOB INFO LIST FOLLOW : " if (type(show_info) == str): for show_info_line in show_info: print show_info_line else: print show_info
def get_job_by_username_with_jobname(_username, _jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) result = json.loads(job_info).get("result") for job_elem in result: show_info = "" for key in filter( lambda _key: (_key.find("jobName") >= 0 or _key.find( "status") >= 0 or _key.find("jobUrl") >= 0 or _key.find( "submitTime") >= 0 or _key.find("clusterType") >= 0), job_elem): if job_elem['account'] == _username: show_info += "%s : %s\n\t" % (key, job_elem[key]) logger.info("\n" + show_info)
def get_job_info_by_jobname_special_k8s(_jobname, _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) if (_param.has_key("job_common_submit_type")): job_common_submit_type = _param["job_common_submit_type"] if (_param.has_key("job_common_is_distributed")): job_common_is_distributed = _param["job_common_is_distributed"] if (_param.has_key("cluster_dispatch_type")): cluster_dispatch_type = _param["cluster_dispatch_type"] if (_param.has_key("job_query_command")): # job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po""" job_query_command = "sh %s po" % _param["job_query_command"] if (_param.has_key("job_common_is_docker")): job_common_is_docker = _param.get("job_common_is_docker") if (job_common_submit_type == "tensorflow" and job_common_is_distributed == "1" and cluster_dispatch_type == "k8s"): query_status_param = {} query_status_param["job_name"] = _jobname query_status_param["command"] = job_query_command query_status_param["job_common_is_docker"] = job_common_is_docker # url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname) host_url = gen_host(_param) + "/queryJobStatus.do" job_info = post(host_url, query_status_param) elif (job_common_submit_type == "spark" and job_common_is_distributed == "1" and cluster_dispatch_type == "yarn"): query_status_param = {} query_status_param["job_name"] = _jobname query_status_param["command"] = "sh " + _param["job_query_command"] query_status_param["job_common_is_docker"] = job_common_is_docker # url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname) host_url = gen_host(_param) + "/queryJobStatus.do" job_info = post(host_url, query_status_param) result_info = json.loads(job_info).get("result") if (len(result_info) <= 0): logger.error("no result info received. PLEASE CHECK JOB_NAME. ") print("no result info received. PLEASE CHECK JOB_NAME. ") exit(1) show_info = result_info[0] split_show_info = show_info.split("proxy") if (len(split_show_info) > 1): EMR_MASTER = "http://10.87.49.221:8088/cluster/" APP_INFO = "app/" + split_show_info[-1] show_info = EMR_MASTER + APP_INFO logger.debug("SHOW_INFO : " + show_info) print u"TASK信息 : \n\t%s" % show_info print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=0&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], _jobname, job_common_is_docker) print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=1&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], _jobname, job_common_is_docker) exit(0) else: url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname job_info = get(url) logger.debug(job_info) show_info = format_job_info(job_info) print "JOB INFO LIST FOLLOW : " for show_info_line in show_info: print u"TASK信息 : \n\t%s" % show_info_line print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+0+%s&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], show_info_line.split(" ")[0], _jobname, job_common_is_docker) print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+1+%s&job_name=%s&job_common_is_docker=%s" % ( host_name, _param["job_log_command"], show_info_line.split(" ")[0], _jobname, job_common_is_docker)