def client_kill_job(_job_name="Empty",
_job_stop_command="Empty",
_param_map={}):
logger = wei_logger("wei_kill_job")
logger.info("Job Stop Command: %s" % _job_stop_command)
logger.info("Job Name: %s" % _job_name)
# if(_param_map.has_key("client_change_host") and
# _param_map["client_change_host"] == "true"):
# client_center_host_name = get_or_else(_param_map,
# "client_center_host_name",
# "http://controlcenter.ds.sina.com.cn")
# if(_param_map.has_key("client_control_center_version")):
# client_control_center_version = get_or_else(_param_map,
# "client_control_center_version",
# "controlCenter-1.0.0")
client_center_host_name = url_helper.gen_host(_param_map)
if (int(_param_map.get("job_common_is_docker")) == 0):
url = url_helper.gen_url_kill_job(
_client_center_host_name=client_center_host_name,
_job_name=_job_name,
_job_stop_command=_job_stop_command)
post_interface = client_center_host_name + "/killJob.do"
req_info = http_helper.post(post_interface, _param_map)
elif (int(_param_map.get("job_common_is_docker")) == 1):
post_interface = client_center_host_name + "/killJob.do"
req_info = http_helper.post(post_interface, _param_map)
else:
logger.error("Job config error: 'Job_common_is_docker'")
exit(1)
kill_request_info_check(req_info)
def get_or_warn(_dict = {},
_key = '' ):
logger = logger_helper.wei_logger("dict_helper")
value = _dict.get(_key)
if(value == None):
logger.warn("key : %s Not Found !!" % _key)
exit(1)
else:
return value
def client_rsync(_src_name="", _dest_name="", _target_repo=""):
logger = wei_logger("client_rsync")
logger.info("src_name : %s" % _src_name)
if (len(_target_repo) == 0):
data_cmd_show = "rsync -vzrtopg --progress %s fileCenter::backup/data/" % _src_name
logger.info("rsync data_cmd: %s" % data_cmd_show)
data_cmd = "rsync -vzrtopg --progress %s 10.77.29.68::backup/data/" % _src_name
else:
data_cmd = "rsync -vzrtopg --progress %s %s%s" % (
_src_name, _target_repo, _dest_name)
logger.info("rsync data_cmd: %s" % data_cmd)
sys_helper.sys_run(data_cmd)
def client_main(_args):
logger = logger_helper.wei_logger("client_main")
# print _args
# ['./bin/../libs/client_origin.py', 'kill', 'dcgan-gpu3'] - 0-based
if (len(_args) < 2):
logger.error(show_usage.global_usage())
exit(1)
if (_args[1].lower() == "submit"):
wei_submit.client_submit(_args[2:])
elif (_args[1].lower() == "rsync"):
if (len(_args) < 4):
logger.error(
"weicmd rsync ${src_dir} ${dest_dir} ${target_reposity}")
elif (len(_args) == 5):
wei_rsync.client_rsync(_args[2], _args[3], _args[4])
elif (len(_args) == 4):
wei_rsync.client_rsync(_args[2], _args[3])
elif (_args[1].lower() == "query"):
if (len(_args) < 3):
logger.error("weicmd query ${subcommand} - supported: cluster,job")
elif (_args[2].lower() == "cluster"):
wei_query_cluster.query_cluster_info_client(_args[2:])
elif (_args[2].lower() == "job"):
wei_query_job.query_job_info_client(_args[2:])
else:
wei_query_job.query_job_info_client(["job"] + _args[2:])
elif (_args[1].lower() == "kill"):
if (len(_args) < 3):
logger.error(
"weicmd kill ${job_name} ${cluster_type}(can be empty) ")
if (len(_args) <= 3):
wei_kill_job.client_kill_job(_args[2])
else:
if (_args[2][0] == "-"):
params = init_get_opts.init_get_opts(_args[2:])
if (params.has_key("job_command")
or params.has_key("job_stop_command")):
wei_kill_job.client_kill_job(
_job_name=params["job_name"],
_job_stop_command=params["job_stop_command"],
_param_map=params)
else:
wei_kill_job.client_kill_job(params["job_name"])
else:
wei_kill_job.client_kill_job(_args[2], _args[3])
else:
logger.error(show_usage.global_usage())
exit(1)
def request_info_check(http_response_code = "", logger = wei_logger()):
if(len(http_response_code.strip()) == 0 ):
show_success_info()
elif(http_response_code.strip()[0] == "{"):
http_response_code_json_map = json.loads(http_response_code)
if(http_response_code_json_map.has_key("message")):
if( http_response_code_json_map.get("message").find(u'success') >= 0 ):
show_success_info()
else:
logger.error(http_response_code_json_map["message"])
elif( http_response_code_json_map.has_key("errCode") ):
if( http_response_code_json_map.get("errCode") == u''):
show_success_info()
else:
logger.error(http_response_code_json_map["errMsg"])
else:
logger.error(http_response_code_json_map["errMsg"])
else:
logger.error(http_response_code)
# !/usr/bin/python
# -*- coding: UTF-8 -*-
import sys
from com.weibo.reflection.libs.ref_load import *
from com.weibo.tools.logger_helper import wei_logger
logger = wei_logger("cluster_client")
def gen_module_name(_name = ""):
return "com.weibo.modules.cluster.weihelper.wei_" + _name
def run(_args):
_func_name = sys._getframe().f_code.co_name
_file_name = __file__.split("/")[-1].split(".py")[0]
logger.info("this is " + _file_name + " " + _func_name + " function.")
logger.debug(_args)
ref_load(gen_module_name(_args[0]))(_args[1:])
if __name__ == "__main__":
run(sys.argv[2:])
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json
import sys
import urllib2
from com.weibo.tools.url_helper import gen_host
from com.weibo.tools import logger_helper
from com.weibo.tools.init_get_opts import query_job_get_opts
logger = logger_helper.wei_logger("wei_query_log")
def query_log_client(_args):
if (len(_args) < 2):
# print "[WARN] : Query job Informations Need Parameters."
# print " OR Query All job Informations Need Parameter 'all'."
# print " OR Query job Informations by Job Name Need Parameter '$job_name'."
# print """ OR "-f" "--conf_file" """
# print """ "-j", "--job_name" """
# print """ "-n", "--user_name" """
# print """ "-t", "--query_type(username OR jobname)(k8s only supported jobname)" """
exit(1)
if __name__ == '__main__':
query_log_client(sys.argv)
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import urllib
import urllib2
from com.weibo.tools import logger_helper
logger = logger_helper.wei_logger("http_helper")
#GET：
def get(_url):
try:
logger.debug("URL : " + _url) #输出查看编码后的数据格式
response = urllib2.urlopen(_url)
req_info = response.read()
logger.debug("%s" % req_info)
return req_info
except (urllib2.HTTPError, Exception), e:
logger.error(e)
exit(1)
#POST：
def post(_url, _values):
try:
data = urllib.urlencode(_values) #适用urllib对数据进行格式化编码
logger.debug("URL : " + _url) #输出查看编码后的数据格式
logger.debug("DATA : " + data) #输出查看编码后的数据格式
req = urllib2.Request(_url, data) #生成页面请求的完整数据
response = urllib2.urlopen(req) #发送页面请求
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json
import sys
import urllib2
from com.weibo.tools.url_helper import gen_host
from com.weibo.tools import logger_helper
from com.weibo.tools.utils import privilege_completion
from com.weibo.tools.init_get_opts import query_get_opts
logger = logger_helper.wei_logger("query_cluster_info")
def useful_info(_key=""):
return (_key.find("version") >= 0 or _key.find("clusterName") >= 0
or _key.find("uiUrl") >= 0 or _key.find("clusterType") >= 0)
def get_all_cluster(_cluster_name="", _param={}):
if (len(_param) == 0):
host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0"
else:
host_name = gen_host(_param)
url = "%s/query.do?op=query&queryType=cluster&clusterName=%s" % (
host_name, _cluster_name)
response = urllib2.urlopen(url)
cluster_info = response.read()
result = json.loads(cluster_info).get("result")
for cluster_elem in result:
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import getopt
import sys
from com.weibo.tools import logger_helper
from com.weibo.tools.show_usage import *
logger = logger_helper.wei_logger("init_get_opts")
def load_config(_file_name=""):
logger.info("Read Config File with file_name: %s" % _file_name)
file_object = open(_file_name)
try:
all_the_config = file_object.read()
finally:
file_object.close()
lines = all_the_config.split('\n')
rst_dict = {}
for line in lines:
config = line.split('#')[0].split('=')
if (line.startswith('#') != True and len(config) >= 2):
config_name = config[0].strip()
config_value = config[1].strip()
rst_dict[config_name] = config_value
return rst_dict
# load_config(_file_name="./weiclient.conf.template")
# !/usr/bin/python
# -*- coding: UTF-8 -*-
# Created by enzhao on 2017/12/11.
import getopt
import os, sys
from com.weibo.tools import logger_helper
from com.weibo.tools.show_usage import *
from com.weibo.tools import dict_helper
from com.weibo.tools import init_get_opts
logger = logger_helper.wei_logger("args_helper")
def paramter_convertor(_param_dict={}):
_param_dict["job_name"] = dict_helper.get_or_else(
_param_dict, "j",
dict_helper.get_or_else(_param_dict, "job_name", "ERROR_JOB_NAME"))
_param_dict["center_user_name"] = dict_helper.get_or_else(
_param_dict, "n",
dict_helper.get_or_else(_param_dict, "center_user_name",
"ERROR_USER_NAME"))
_param_dict["job_key"] = dict_helper.get_or_else(
_param_dict, "k",
dict_helper.get_or_else(_param_dict, "job_key", "EMPTY_JOB_KEY"))
config_path = _param_dict.get("config") if _param_dict.has_key(
"config") else _param_dict.get("f")
core_config_file_path = os.path.dirname(os.path.abspath(
sys.argv[0])) + "/../conf/weiclient.conf.core.site.template"
core_dict = init_get_opts.load_config(core_config_file_path)
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import json
import sys
import urllib2
from com.weibo.tools.url_helper import gen_host
from com.weibo.tools import logger_helper
from com.weibo.tools.init_get_opts import query_job_get_opts
from com.weibo.tools.http_helper import post, get
logger = logger_helper.wei_logger("query_job_info")
def useful_info(_key=""):
return (_key.find("version") >= 0 or _key.find("jobName") >= 0
or _key.find("uiUrl") >= 0 or _key.find("jobType") >= 0)
def format_job_info(_job_info=""):
result = json.loads(_job_info).get("result")
show_info = ""
for job_elem in result:
for key in filter(
lambda _key: (_key.find("jobName") >= 0 or _key.find(
"status") >= 0 or _key.find("jobUrl") >= 0 or _key.find(
"submitTime") >= 0 or _key.find("clusterType") >= 0),
job_elem):
show_info += "%s : %s\n\t" % (key, job_elem[key])
if (len(show_info) == 0):
# !/usr/bin/python
# -*- coding: UTF-8 -*-
# Created by enzhao on 2017/12/11.
import json, os, urllib2
from com.weibo.tools import logger_helper
from com.weibo.tools.http_helper import get
from com.weibo.tools.show_usage import *
from com.weibo.tools.sys_helper import *
logger = logger_helper.wei_logger("update_helper")
def check_update(_version="0.5.2.0", _size=""):
url = "http://10.77.29.69:8080/controlCenter-1.0.0/api/clientUpdate"
url = "http://10.77.29.69:8080/waic/api/clientUpdate"
req_dict = json.loads(get(url))
new_result = req_dict.get("result")[0]
new_version = new_result.get("version")
new_size = new_result.get("size")
return not (_version == new_version and _size == new_size)
def download_weiclient_latest(_url="", _weiclient_path="../"):
download_url = _url
try:
logger.debug("URL : " + _url) #输出查看编码后的数据格式
logger.debug("DOWNLOAD_URL : " + download_url) #输出查看编码后的数据格式
response = urllib2.urlopen(_url)
remote_tar_bytes = response.read()
#!/usr/bin/python
# -*- coding: UTF-8 -*-
from com.weibo.tools import logger_helper
from com.weibo.tools.dict_helper import *
logger = logger_helper.wei_logger("url_helper")
# generate url by config dictionary
def gen_url_by_dict(_dict, _last_dir):
# config host_name
client_center_host_name = 'http://controlcenter.ds.sina.com.cn'
change_host = get_or_str(_dict, "change_host", False)
if (change_host == 'true'):
client_center_host_name = get_or_str(
_dict, "host_name", 'http://controlcenter.ds.sina.com.cn')
# config control_center_version
client_control_center_version = 'controlCenter-1.0.0'
if (get_or_else(_dict, "control_center_version") != None):
client_control_center_version = get_or_else(
_dict, "client_control_center_version")
# read user_name
center_user_name = get_or_str(_dict, "center_user_name", "Empty").strip()
# read job_name
job_name = get_or_str(_dict, "job_name", "Empty").strip()
# read cluster_version
cluster_version = get_or_str(_dict, "cluster_version", "1.3.0").strip()
# read cluster_type
cluster_type = get_or_str(_dict, "cluster_type", "Empty").strip()
# read cluster_name
#!/usr/bin/python
# -*- coding: UTF-8 -*-
# model(client)
from com.weibo.tools import sys_helper
from com.weibo.tools.logger_helper import wei_logger
logger = wei_logger("client_rsync")
def client_rsync(_src_name = "",
_dest_name = "",
_target_repo = ""):
logger.info("src_name : %s" % _src_name)
if(len(_target_repo) == 0):
data_cmd_show = "rsync -vzrtopg --progress %s fileCenter::backup/data/" % _src_name
logger.info("rsync data_cmd: %s" % data_cmd_show)
data_cmd = "rsync -vzrtopg --progress %s 10.77.29.68::backup/data/" % _src_name
else:
data_cmd = "rsync -vzrtopg --progress %s %s%s" % (_src_name, _target_repo, _dest_name)
logger.info("rsync data_cmd: %s" % data_cmd)
sys_helper.sys_run(data_cmd)
def run(_args = []):
if(len(_args) == 3):
client_rsync(_args[0], _args[1], _args[2])
else:
logger.error("client_rsync need specify three parameters! EXIT.")
exit(1)