def get_job_log_by_jobname_special_k8s(_jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
if (_param.has_key("job_common_submit_type")):
job_common_submit_type = _param["job_common_submit_type"]
if (_param.has_key("job_common_is_distributed")):
job_common_is_distributed = _param["job_common_is_distributed"]
if (_param.has_key("cluster_dispatch_type")):
cluster_dispatch_type = _param["cluster_dispatch_type"]
if (_param.has_key("job_query_command")):
# job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po"""
job_query_command = "sh %s %s" % (_param["job_log_command"],
_param["query_name"])
if (job_common_submit_type == "tensorflow"
and job_common_is_distributed == "1"
and cluster_dispatch_type == "k8s"):
query_status_param = {}
query_status_param["job_name"] = _jobname
query_status_param["command"] = job_query_command
# url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname)
host_url = gen_host(_param) + "/queryJobLog.do"
job_info = post(host_url, query_status_param)
else:
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
logger.debug(job_info)
print "JOB LOG LIST FOLLOW : "
print job_info
def check_update(_version="0.5.2.0", _size=""):
url = "http://10.77.29.69:8080/controlCenter-1.0.0/api/clientUpdate"
url = "http://10.77.29.69:8080/waic/api/clientUpdate"
req_dict = json.loads(get(url))
new_result = req_dict.get("result")[0]
new_version = new_result.get("version")
new_size = new_result.get("size")
return not (_version == new_version and _size == new_size)
def get_job_info_by_jobname(_jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
show_info = format_job_info(job_info)
print "JOB INFO LIST FOLLOW : "
for show_info_line in show_info:
print show_info_line
def get_all_job(_job_name="", _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = '%s/query.do?op=query&query_type=job&job_name=%s' % (host_name,
_job_name)
job_info = get(url)
show_info = format_job_info(job_info)
print "JOB INFO LIST FOLLOW : "
if (type(show_info) == str):
for show_info_line in show_info:
print show_info_line
else:
print show_info
def get_job_by_username_with_jobname(_username, _jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
result = json.loads(job_info).get("result")
for job_elem in result:
show_info = ""
for key in filter(
lambda _key: (_key.find("jobName") >= 0 or _key.find(
"status") >= 0 or _key.find("jobUrl") >= 0 or _key.find(
"submitTime") >= 0 or _key.find("clusterType") >= 0),
job_elem):
if job_elem['account'] == _username:
show_info += "%s : %s\n\t" % (key, job_elem[key])
logger.info("\n" + show_info)
def get_job_info_by_jobname_special_k8s(_jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
if (_param.has_key("job_common_submit_type")):
job_common_submit_type = _param["job_common_submit_type"]
if (_param.has_key("job_common_is_distributed")):
job_common_is_distributed = _param["job_common_is_distributed"]
if (_param.has_key("cluster_dispatch_type")):
cluster_dispatch_type = _param["cluster_dispatch_type"]
if (_param.has_key("job_query_command")):
# job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po"""
job_query_command = "sh %s po" % _param["job_query_command"]
if (_param.has_key("job_common_is_docker")):
job_common_is_docker = _param.get("job_common_is_docker")
if (job_common_submit_type == "tensorflow"
and job_common_is_distributed == "1"
and cluster_dispatch_type == "k8s"):
query_status_param = {}
query_status_param["job_name"] = _jobname
query_status_param["command"] = job_query_command
query_status_param["job_common_is_docker"] = job_common_is_docker
# url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname)
host_url = gen_host(_param) + "/queryJobStatus.do"
job_info = post(host_url, query_status_param)
elif (job_common_submit_type == "spark"
and job_common_is_distributed == "1"
and cluster_dispatch_type == "yarn"):
query_status_param = {}
query_status_param["job_name"] = _jobname
query_status_param["command"] = "sh " + _param["job_query_command"]
query_status_param["job_common_is_docker"] = job_common_is_docker
# url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname)
host_url = gen_host(_param) + "/queryJobStatus.do"
job_info = post(host_url, query_status_param)
result_info = json.loads(job_info).get("result")
if (len(result_info) <= 0):
logger.error("no result info received. PLEASE CHECK JOB_NAME. ")
print("no result info received. PLEASE CHECK JOB_NAME. ")
exit(1)
show_info = result_info[0]
split_show_info = show_info.split("proxy")
if (len(split_show_info) > 1):
EMR_MASTER = "http://10.87.49.221:8088/cluster/"
APP_INFO = "app/" + split_show_info[-1]
show_info = EMR_MASTER + APP_INFO
logger.debug("SHOW_INFO : " + show_info)
print u"TASK信息 : \n\t%s" % show_info
print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=0&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], _jobname,
job_common_is_docker)
print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=1&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], _jobname,
job_common_is_docker)
exit(0)
else:
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
logger.debug(job_info)
show_info = format_job_info(job_info)
print "JOB INFO LIST FOLLOW : "
for show_info_line in show_info:
print u"TASK信息 : \n\t%s" % show_info_line
print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+0+%s&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], show_info_line.split(" ")[0],
_jobname, job_common_is_docker)
print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+1+%s&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], show_info_line.split(" ")[0],
_jobname, job_common_is_docker)