def client_kill_job(_job_name="Empty", _job_stop_command="Empty", _param_map={}): logger = wei_logger("wei_kill_job") logger.info("Job Stop Command: %s" % _job_stop_command) logger.info("Job Name: %s" % _job_name) # if(_param_map.has_key("client_change_host") and # _param_map["client_change_host"] == "true"): # client_center_host_name = get_or_else(_param_map, # "client_center_host_name", # "http://controlcenter.ds.sina.com.cn") # if(_param_map.has_key("client_control_center_version")): # client_control_center_version = get_or_else(_param_map, # "client_control_center_version", # "controlCenter-1.0.0") client_center_host_name = url_helper.gen_host(_param_map) if (int(_param_map.get("job_common_is_docker")) == 0): url = url_helper.gen_url_kill_job( _client_center_host_name=client_center_host_name, _job_name=_job_name, _job_stop_command=_job_stop_command) post_interface = client_center_host_name + "/killJob.do" req_info = http_helper.post(post_interface, _param_map) elif (int(_param_map.get("job_common_is_docker")) == 1): post_interface = client_center_host_name + "/killJob.do" req_info = http_helper.post(post_interface, _param_map) else: logger.error("Job config error: 'Job_common_is_docker'") exit(1) kill_request_info_check(req_info)
def get_or_warn(_dict = {}, _key = '' ): logger = logger_helper.wei_logger("dict_helper") value = _dict.get(_key) if(value == None): logger.warn("key : %s Not Found !!" % _key) exit(1) else: return value
def client_rsync(_src_name="", _dest_name="", _target_repo=""): logger = wei_logger("client_rsync") logger.info("src_name : %s" % _src_name) if (len(_target_repo) == 0): data_cmd_show = "rsync -vzrtopg --progress %s fileCenter::backup/data/" % _src_name logger.info("rsync data_cmd: %s" % data_cmd_show) data_cmd = "rsync -vzrtopg --progress %s 10.77.29.68::backup/data/" % _src_name else: data_cmd = "rsync -vzrtopg --progress %s %s%s" % ( _src_name, _target_repo, _dest_name) logger.info("rsync data_cmd: %s" % data_cmd) sys_helper.sys_run(data_cmd)
def client_main(_args): logger = logger_helper.wei_logger("client_main") # print _args # ['./bin/../libs/client_origin.py', 'kill', 'dcgan-gpu3'] - 0-based if (len(_args) < 2): logger.error(show_usage.global_usage()) exit(1) if (_args[1].lower() == "submit"): wei_submit.client_submit(_args[2:]) elif (_args[1].lower() == "rsync"): if (len(_args) < 4): logger.error( "weicmd rsync ${src_dir} ${dest_dir} ${target_reposity}") elif (len(_args) == 5): wei_rsync.client_rsync(_args[2], _args[3], _args[4]) elif (len(_args) == 4): wei_rsync.client_rsync(_args[2], _args[3]) elif (_args[1].lower() == "query"): if (len(_args) < 3): logger.error("weicmd query ${subcommand} - supported: cluster,job") elif (_args[2].lower() == "cluster"): wei_query_cluster.query_cluster_info_client(_args[2:]) elif (_args[2].lower() == "job"): wei_query_job.query_job_info_client(_args[2:]) else: wei_query_job.query_job_info_client(["job"] + _args[2:]) elif (_args[1].lower() == "kill"): if (len(_args) < 3): logger.error( "weicmd kill ${job_name} ${cluster_type}(can be empty) ") if (len(_args) <= 3): wei_kill_job.client_kill_job(_args[2]) else: if (_args[2][0] == "-"): params = init_get_opts.init_get_opts(_args[2:]) if (params.has_key("job_command") or params.has_key("job_stop_command")): wei_kill_job.client_kill_job( _job_name=params["job_name"], _job_stop_command=params["job_stop_command"], _param_map=params) else: wei_kill_job.client_kill_job(params["job_name"]) else: wei_kill_job.client_kill_job(_args[2], _args[3]) else: logger.error(show_usage.global_usage()) exit(1)
def request_info_check(http_response_code = "", logger = wei_logger()): if(len(http_response_code.strip()) == 0 ): show_success_info() elif(http_response_code.strip()[0] == "{"): http_response_code_json_map = json.loads(http_response_code) if(http_response_code_json_map.has_key("message")): if( http_response_code_json_map.get("message").find(u'success') >= 0 ): show_success_info() else: logger.error(http_response_code_json_map["message"]) elif( http_response_code_json_map.has_key("errCode") ): if( http_response_code_json_map.get("errCode") == u''): show_success_info() else: logger.error(http_response_code_json_map["errMsg"]) else: logger.error(http_response_code_json_map["errMsg"]) else: logger.error(http_response_code)
# !/usr/bin/python # -*- coding: UTF-8 -*- import sys from com.weibo.reflection.libs.ref_load import * from com.weibo.tools.logger_helper import wei_logger logger = wei_logger("cluster_client") def gen_module_name(_name = ""): return "com.weibo.modules.cluster.weihelper.wei_" + _name def run(_args): _func_name = sys._getframe().f_code.co_name _file_name = __file__.split("/")[-1].split(".py")[0] logger.info("this is " + _file_name + " " + _func_name + " function.") logger.debug(_args) ref_load(gen_module_name(_args[0]))(_args[1:]) if __name__ == "__main__": run(sys.argv[2:])
#!/usr/bin/python # -*- coding: UTF-8 -*- import json import sys import urllib2 from com.weibo.tools.url_helper import gen_host from com.weibo.tools import logger_helper from com.weibo.tools.init_get_opts import query_job_get_opts logger = logger_helper.wei_logger("wei_query_log") def query_log_client(_args): if (len(_args) < 2): # print "[WARN] : Query job Informations Need Parameters." # print " OR Query All job Informations Need Parameter 'all'." # print " OR Query job Informations by Job Name Need Parameter '$job_name'." # print """ OR "-f" "--conf_file" """ # print """ "-j", "--job_name" """ # print """ "-n", "--user_name" """ # print """ "-t", "--query_type(username OR jobname)(k8s only supported jobname)" """ exit(1) if __name__ == '__main__': query_log_client(sys.argv)
#!/usr/bin/env python # -*- coding:utf-8 -*- import urllib import urllib2 from com.weibo.tools import logger_helper logger = logger_helper.wei_logger("http_helper") #GET: def get(_url): try: logger.debug("URL : " + _url) #输出查看编码后的数据格式 response = urllib2.urlopen(_url) req_info = response.read() logger.debug("%s" % req_info) return req_info except (urllib2.HTTPError, Exception), e: logger.error(e) exit(1) #POST: def post(_url, _values): try: data = urllib.urlencode(_values) #适用urllib对数据进行格式化编码 logger.debug("URL : " + _url) #输出查看编码后的数据格式 logger.debug("DATA : " + data) #输出查看编码后的数据格式 req = urllib2.Request(_url, data) #生成页面请求的完整数据 response = urllib2.urlopen(req) #发送页面请求
#!/usr/bin/python # -*- coding: UTF-8 -*- import json import sys import urllib2 from com.weibo.tools.url_helper import gen_host from com.weibo.tools import logger_helper from com.weibo.tools.utils import privilege_completion from com.weibo.tools.init_get_opts import query_get_opts logger = logger_helper.wei_logger("query_cluster_info") def useful_info(_key=""): return (_key.find("version") >= 0 or _key.find("clusterName") >= 0 or _key.find("uiUrl") >= 0 or _key.find("clusterType") >= 0) def get_all_cluster(_cluster_name="", _param={}): if (len(_param) == 0): host_name = "http://controlcenter.ds.sina.com.cn/controlCenter-1.0.0" else: host_name = gen_host(_param) url = "%s/query.do?op=query&queryType=cluster&clusterName=%s" % ( host_name, _cluster_name) response = urllib2.urlopen(url) cluster_info = response.read() result = json.loads(cluster_info).get("result") for cluster_elem in result:
#!/usr/bin/python # -*- coding: UTF-8 -*- import getopt import sys from com.weibo.tools import logger_helper from com.weibo.tools.show_usage import * logger = logger_helper.wei_logger("init_get_opts") def load_config(_file_name=""): logger.info("Read Config File with file_name: %s" % _file_name) file_object = open(_file_name) try: all_the_config = file_object.read() finally: file_object.close() lines = all_the_config.split('\n') rst_dict = {} for line in lines: config = line.split('#')[0].split('=') if (line.startswith('#') != True and len(config) >= 2): config_name = config[0].strip() config_value = config[1].strip() rst_dict[config_name] = config_value return rst_dict # load_config(_file_name="./weiclient.conf.template")
# !/usr/bin/python # -*- coding: UTF-8 -*- # Created by enzhao on 2017/12/11. import getopt import os, sys from com.weibo.tools import logger_helper from com.weibo.tools.show_usage import * from com.weibo.tools import dict_helper from com.weibo.tools import init_get_opts logger = logger_helper.wei_logger("args_helper") def paramter_convertor(_param_dict={}): _param_dict["job_name"] = dict_helper.get_or_else( _param_dict, "j", dict_helper.get_or_else(_param_dict, "job_name", "ERROR_JOB_NAME")) _param_dict["center_user_name"] = dict_helper.get_or_else( _param_dict, "n", dict_helper.get_or_else(_param_dict, "center_user_name", "ERROR_USER_NAME")) _param_dict["job_key"] = dict_helper.get_or_else( _param_dict, "k", dict_helper.get_or_else(_param_dict, "job_key", "EMPTY_JOB_KEY")) config_path = _param_dict.get("config") if _param_dict.has_key( "config") else _param_dict.get("f") core_config_file_path = os.path.dirname(os.path.abspath( sys.argv[0])) + "/../conf/weiclient.conf.core.site.template" core_dict = init_get_opts.load_config(core_config_file_path)
#!/usr/bin/python # -*- coding: UTF-8 -*- import json import sys import urllib2 from com.weibo.tools.url_helper import gen_host from com.weibo.tools import logger_helper from com.weibo.tools.init_get_opts import query_job_get_opts from com.weibo.tools.http_helper import post, get logger = logger_helper.wei_logger("query_job_info") def useful_info(_key=""): return (_key.find("version") >= 0 or _key.find("jobName") >= 0 or _key.find("uiUrl") >= 0 or _key.find("jobType") >= 0) def format_job_info(_job_info=""): result = json.loads(_job_info).get("result") show_info = "" for job_elem in result: for key in filter( lambda _key: (_key.find("jobName") >= 0 or _key.find( "status") >= 0 or _key.find("jobUrl") >= 0 or _key.find( "submitTime") >= 0 or _key.find("clusterType") >= 0), job_elem): show_info += "%s : %s\n\t" % (key, job_elem[key]) if (len(show_info) == 0):
# !/usr/bin/python # -*- coding: UTF-8 -*- # Created by enzhao on 2017/12/11. import json, os, urllib2 from com.weibo.tools import logger_helper from com.weibo.tools.http_helper import get from com.weibo.tools.show_usage import * from com.weibo.tools.sys_helper import * logger = logger_helper.wei_logger("update_helper") def check_update(_version="0.5.2.0", _size=""): url = "http://10.77.29.69:8080/controlCenter-1.0.0/api/clientUpdate" url = "http://10.77.29.69:8080/waic/api/clientUpdate" req_dict = json.loads(get(url)) new_result = req_dict.get("result")[0] new_version = new_result.get("version") new_size = new_result.get("size") return not (_version == new_version and _size == new_size) def download_weiclient_latest(_url="", _weiclient_path="../"): download_url = _url try: logger.debug("URL : " + _url) #输出查看编码后的数据格式 logger.debug("DOWNLOAD_URL : " + download_url) #输出查看编码后的数据格式 response = urllib2.urlopen(_url) remote_tar_bytes = response.read()
#!/usr/bin/python # -*- coding: UTF-8 -*- from com.weibo.tools import logger_helper from com.weibo.tools.dict_helper import * logger = logger_helper.wei_logger("url_helper") # generate url by config dictionary def gen_url_by_dict(_dict, _last_dir): # config host_name client_center_host_name = 'http://controlcenter.ds.sina.com.cn' change_host = get_or_str(_dict, "change_host", False) if (change_host == 'true'): client_center_host_name = get_or_str( _dict, "host_name", 'http://controlcenter.ds.sina.com.cn') # config control_center_version client_control_center_version = 'controlCenter-1.0.0' if (get_or_else(_dict, "control_center_version") != None): client_control_center_version = get_or_else( _dict, "client_control_center_version") # read user_name center_user_name = get_or_str(_dict, "center_user_name", "Empty").strip() # read job_name job_name = get_or_str(_dict, "job_name", "Empty").strip() # read cluster_version cluster_version = get_or_str(_dict, "cluster_version", "1.3.0").strip() # read cluster_type cluster_type = get_or_str(_dict, "cluster_type", "Empty").strip() # read cluster_name
#!/usr/bin/python # -*- coding: UTF-8 -*- # model(client) from com.weibo.tools import sys_helper from com.weibo.tools.logger_helper import wei_logger logger = wei_logger("client_rsync") def client_rsync(_src_name = "", _dest_name = "", _target_repo = ""): logger.info("src_name : %s" % _src_name) if(len(_target_repo) == 0): data_cmd_show = "rsync -vzrtopg --progress %s fileCenter::backup/data/" % _src_name logger.info("rsync data_cmd: %s" % data_cmd_show) data_cmd = "rsync -vzrtopg --progress %s 10.77.29.68::backup/data/" % _src_name else: data_cmd = "rsync -vzrtopg --progress %s %s%s" % (_src_name, _target_repo, _dest_name) logger.info("rsync data_cmd: %s" % data_cmd) sys_helper.sys_run(data_cmd) def run(_args = []): if(len(_args) == 3): client_rsync(_args[0], _args[1], _args[2]) else: logger.error("client_rsync need specify three parameters! EXIT.") exit(1)