def get_job_log_by_jobname_special_k8s(_jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
if (_param.has_key("job_common_submit_type")):
job_common_submit_type = _param["job_common_submit_type"]
if (_param.has_key("job_common_is_distributed")):
job_common_is_distributed = _param["job_common_is_distributed"]
if (_param.has_key("cluster_dispatch_type")):
cluster_dispatch_type = _param["cluster_dispatch_type"]
if (_param.has_key("job_query_command")):
# job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po"""
job_query_command = "sh %s %s" % (_param["job_log_command"],
_param["query_name"])
if (job_common_submit_type == "tensorflow"
and job_common_is_distributed == "1"
and cluster_dispatch_type == "k8s"):
query_status_param = {}
query_status_param["job_name"] = _jobname
query_status_param["command"] = job_query_command
# url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname)
host_url = gen_host(_param) + "/queryJobLog.do"
job_info = post(host_url, query_status_param)
else:
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
logger.debug(job_info)
print "JOB LOG LIST FOLLOW : "
print job_info
def client_kill_job(_job_name="Empty",
_job_stop_command="Empty",
_param_map={}):
logger = wei_logger("wei_kill_job")
logger.info("Job Stop Command: %s" % _job_stop_command)
logger.info("Job Name: %s" % _job_name)
# if(_param_map.has_key("client_change_host") and
# _param_map["client_change_host"] == "true"):
# client_center_host_name = get_or_else(_param_map,
# "client_center_host_name",
# "http://controlcenter.ds.sina.com.cn")
# if(_param_map.has_key("client_control_center_version")):
# client_control_center_version = get_or_else(_param_map,
# "client_control_center_version",
# "controlCenter-1.0.0")
client_center_host_name = url_helper.gen_host(_param_map)
if (int(_param_map.get("job_common_is_docker")) == 0):
url = url_helper.gen_url_kill_job(
_client_center_host_name=client_center_host_name,
_job_name=_job_name,
_job_stop_command=_job_stop_command)
post_interface = client_center_host_name + "/killJob.do"
req_info = http_helper.post(post_interface, _param_map)
elif (int(_param_map.get("job_common_is_docker")) == 1):
post_interface = client_center_host_name + "/killJob.do"
req_info = http_helper.post(post_interface, _param_map)
else:
logger.error("Job config error: 'Job_common_is_docker'")
exit(1)
kill_request_info_check(req_info)
def get_group_info_by_username(_username, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/account_group.do?account=" % host_name + _username
response = urllib2.urlopen(url)
try:
cluster_info = json.loads(response.read())[0]
except (IndexError, StandardError), e:
logger.error("User Name %s has Wrong!!" % _username)
def get_job_info_by_jobname(_jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
show_info = format_job_info(job_info)
print "JOB INFO LIST FOLLOW : "
for show_info_line in show_info:
print show_info_line
def get_all_job(_job_name="", _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = '%s/query.do?op=query&query_type=job&job_name=%s' % (host_name,
_job_name)
job_info = get(url)
show_info = format_job_info(job_info)
print "JOB INFO LIST FOLLOW : "
if (type(show_info) == str):
for show_info_line in show_info:
print show_info_line
else:
print show_info
def get_job_by_username_with_jobname(_username, _jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
result = json.loads(job_info).get("result")
for job_elem in result:
show_info = ""
for key in filter(
lambda _key: (_key.find("jobName") >= 0 or _key.find(
"status") >= 0 or _key.find("jobUrl") >= 0 or _key.find(
"submitTime") >= 0 or _key.find("clusterType") >= 0),
job_elem):
if job_elem['account'] == _username:
show_info += "%s : %s\n\t" % (key, job_elem[key])
logger.info("\n" + show_info)
def get_all_cluster(_cluster_name="", _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/query.do?op=query&queryType=cluster&clusterName=%s" % (
host_name, _cluster_name)
response = urllib2.urlopen(url)
cluster_info = response.read()
result = json.loads(cluster_info).get("result")
for cluster_elem in result:
show_info = ""
for key in filter(
lambda _key:
(_key.find("version") >= 0 or _key.find("clusterName") >= 0 or _key
.find("uiUrl") >= 0 or _key.find("clusterType") >= 0),
cluster_elem):
show_info += "%s : %s\n\t" % (key, cluster_elem[key])
logger.info("\n" + show_info)
def get_job_info_by_jobname_special_k8s(_jobname, _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
if (_param.has_key("job_common_submit_type")):
job_common_submit_type = _param["job_common_submit_type"]
if (_param.has_key("job_common_is_distributed")):
job_common_is_distributed = _param["job_common_is_distributed"]
if (_param.has_key("cluster_dispatch_type")):
cluster_dispatch_type = _param["cluster_dispatch_type"]
if (_param.has_key("job_query_command")):
# job_query_command = "sh%20" + _param["job_query_command"] + """%20""" + """po"""
job_query_command = "sh %s po" % _param["job_query_command"]
if (_param.has_key("job_common_is_docker")):
job_common_is_docker = _param.get("job_common_is_docker")
if (job_common_submit_type == "tensorflow"
and job_common_is_distributed == "1"
and cluster_dispatch_type == "k8s"):
query_status_param = {}
query_status_param["job_name"] = _jobname
query_status_param["command"] = job_query_command
query_status_param["job_common_is_docker"] = job_common_is_docker
# url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname)
host_url = gen_host(_param) + "/queryJobStatus.do"
job_info = post(host_url, query_status_param)
elif (job_common_submit_type == "spark"
and job_common_is_distributed == "1"
and cluster_dispatch_type == "yarn"):
query_status_param = {}
query_status_param["job_name"] = _jobname
query_status_param["command"] = "sh " + _param["job_query_command"]
query_status_param["job_common_is_docker"] = job_common_is_docker
# url = "%s/queryJobStatus.do?command=%s&jobName=%s" % (host_name, job_query_command, _jobname)
host_url = gen_host(_param) + "/queryJobStatus.do"
job_info = post(host_url, query_status_param)
result_info = json.loads(job_info).get("result")
if (len(result_info) <= 0):
logger.error("no result info received. PLEASE CHECK JOB_NAME. ")
print("no result info received. PLEASE CHECK JOB_NAME. ")
exit(1)
show_info = result_info[0]
split_show_info = show_info.split("proxy")
if (len(split_show_info) > 1):
EMR_MASTER = "http://10.87.49.221:8088/cluster/"
APP_INFO = "app/" + split_show_info[-1]
show_info = EMR_MASTER + APP_INFO
logger.debug("SHOW_INFO : " + show_info)
print u"TASK信息 : \n\t%s" % show_info
print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=0&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], _jobname,
job_common_is_docker)
print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s&log_type=1&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], _jobname,
job_common_is_docker)
exit(0)
else:
url = "%s/query.do?op=query&query_type=job&job_name=" % host_name + _jobname
job_info = get(url)
logger.debug(job_info)
show_info = format_job_info(job_info)
print "JOB INFO LIST FOLLOW : "
for show_info_line in show_info:
print u"TASK信息 : \n\t%s" % show_info_line
print u"当前TASK标准日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+0+%s&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], show_info_line.split(" ")[0],
_jobname, job_common_is_docker)
print u"当前TASK用户日志输出链接 : \n\t%s/queryJobLog.do?command=sh+%s+1+%s&job_name=%s&job_common_is_docker=%s" % (
host_name, _param["job_log_command"], show_info_line.split(" ")[0],
_jobname, job_common_is_docker)
