def add_task(self, service_name, command):
"""
current supported commands are: stop, restart, undeploy_tars, patch_tars
"""
utilities.log_debug("add_task for service %s, command is %s" %
(service_name, command))
(ret, server_id) = self.get_server_id(service_name, self.deploy_ip)
if ret is False:
utilities.log_error(
"%s failed for get server id failed, please check the existence of %s"
% (command, service_name))
return False
items = [{
"server_id": server_id,
"command": command,
"parameters": {}
}]
request_data = {"serial": 'true', "items": items}
response = requests.post(self.add_task_url,
params=self.token_param,
json=request_data)
if TarsService.parse_response("execute command " + command,
response) is False:
utilities.log_error(
"add_task failed for error response, server name: %s, msg: %s"
% (service_name, response.content))
return False
return True
def download_binary(self, binary_name, download_url):
if os.path.exists(self.binary_path) is False:
utilities.mkdir(self.binary_path)
binary_file_path = self.get_required_binary_path(binary_name)
utilities.log_info("Download url: %s" % download_url)
with open(binary_file_path, 'wb') as file:
response = requests.get(download_url, stream=True)
total = response.headers.get('content-length')
if total is None or int(total) < 100000:
utilities.log_error(
"Download binary %s failed, Please check the existence of the binary version %s"
% (binary_name, self.version))
return False
utilities.log_info(
"* Download %s from %s\n* size: %fMB, dst_path: %s" %
(binary_name, download_url, float(total) / float(1000000),
binary_file_path))
downloaded = 0
total = int(total)
for data in response.iter_content(
chunk_size=max(int(total / 1000), 1024 * 1024)):
downloaded += len(data)
file.write(data)
done = int(50 * downloaded / total)
utilities.log_info("Download percent: %d%%" %
(downloaded / total * 100))
sys.stdout.write('\r[{}{}]'.format('█' * done,
'.' * (50 - done)))
sys.stdout.flush()
sys.stdout.write('\n')
utilities.log_info("* Download %s from %s success" %
(binary_name, download_url))
return True
def expand_service_to_given_ip(self, service_config, node_name,
expand_node_ip):
config_generator = ServiceConfigGenerator(self.config,
self.service_type,
service_config,
expand_node_ip)
tars_service = TarsService(self.config.tars_config.tars_url,
self.config.tars_config.tars_token,
self.config.chain_id, expand_node_ip)
# expand the service
obj_name = self.get_service_obj()
obj_list = [obj_name]
expand_node_list = [expand_node_ip]
ret = tars_service.expand_server_with_preview(service_config.name,
node_name,
expand_node_list,
obj_list)
if ret is False:
utilities.log_error(
"expand service failed, app: %s, service: %s, node: %s" %
(self.config.chain_id, service_config.name, expand_node_ip))
return False
# add configuration files
ret = tars_service.add_config_list(config_generator.config_file_list,
service_config.name, expand_node_ip,
config_generator.config_path_list,
True)
if ret is False:
return False
# patch the service
org_service_name = self.get_org_service_name()
return self.upgrade_service_by_config_info(tars_service,
service_config,
org_service_name,
config_generator)
def patch_tars(self, server_id, patch_id):
utilities.log_debug(
"patch tars for application %s, server_id: %s, patch_id: %s" %
(self.app_name, server_id, patch_id))
items = [{
"server_id": server_id,
"command": "patch_tars",
"parameters": {
"patch_id": patch_id,
"bak_flag": 'false',
"update_text": "",
"group_name": ""
}
}]
request_data = {"serial": 'true', "items": items}
response = requests.post(self.add_task_url,
params=self.token_param,
json=request_data)
if TarsService.parse_response("patch tars ", response) is False:
utilities.log_error(
"patch tars failed for error response, server id: %s, msg: %s"
% (server_id, response.content))
return False
utilities.log_debug("patch tars response %s" % response.content)
return True
def expand_server_preview(self, server_name, node_name,
expanded_node_list):
"""
expand the server preview
"""
utilities.log_info(
"expand_server_preview, app: %s, server_name: %s, expanded_node_list: %s"
% (self.app_name, server_name, '.'.join(expanded_node_list)))
request_data = {
"application": self.app_name,
"server_name": server_name,
"node_name": node_name,
"set": "",
"expand_nodes": expanded_node_list,
"enable_set": "false",
"set_name": "",
"set_area": "",
"set_group": "",
"copy_node_config": "false",
"nodeName": []
}
response = requests.post(self.expand_server_preview_url,
params=self.token_param,
json=request_data)
if TarsService.parse_response("expand server preview",
response) is False:
utilities.log_error(
"expand server preview for error response, server name: %s, msg: %s"
% (server_name, response.content))
return False
utilities.log_info("expand server preview response %s" %
response.content)
return True
def get_config_file_id(self, config_file_name, server_name, node_name):
(ret, server_config_id) = self.get_server_config_file_id(
config_file_name, server_name)
if ret is False:
return (False, 0)
params = {
"ticket": self.tars_token,
"config_id": server_config_id,
"level": TarsService.get_level(server_name),
"application": self.app_name,
"server_name": server_name,
"set_name": "",
"set_area": "",
"set_group": ""
}
response = requests.get(self.node_config_file_list_url, params=params)
if TarsService.parse_response(
"query the node config file id for " + config_file_name,
response) is False:
return (False, 0)
result = response.json()
if "data" not in result or len(result["data"]) == 0:
utilities.log_debug("the config %s doesn't exist" %
(config_file_name))
return (False, 0)
# try to find the config file info
for item in result["data"]:
if "filename" in item and item[
"filename"] == config_file_name and item[
"node_name"] == node_name:
return (True, item["id"])
utilities.log_error(
"the node config file %s not found, node: %s, :%s:" %
(config_file_name, node_name, str(result["data"])))
return (False, 0)
def get_server_id(self, service_name, node_name):
# tree_node_id
tree_node_id = "1" + self.app_name + ".5" + service_name
(ret, response) = self.get_server_info(tree_node_id)
if ret is False:
return (False, 0)
if TarsService.parse_response("get server list ", response) is False:
utilities.log_error(
"get server info failed for error response, server name: %s, msg: %s"
% (service_name, response.content))
return (False, 0)
result = response.json()
if 'data' not in result:
return (False, 0)
server_infos = result['data']
for item in server_infos:
if "node_name" in item and len(
node_name) > 0 and item["node_name"] == node_name:
return (True, item["id"])
if "id" in item and len(node_name) == 0:
return (True, item["id"])
return (False, 0)
server_id = result["data"][0]["id"]
return (True, server_id)
def add_server_config_file(self, config_file_name, server_name,
config_file_path, empty_server_config):
content = "\n"
if os.path.exists(config_file_path) is False:
utilities.log_error(
"add service config error:\n the config file %s doesn't exist, service: %s"
% (config_file_path, server_name))
return False
if empty_server_config is False:
try:
fp = open(config_file_path)
content = fp.read()
except OSError as reason:
utilities.log_error(
"load the configuration failed, error: %s" % str(reason))
return False
request_data = {
"level": TarsService.get_level(server_name),
"application": self.app_name,
"server_name": server_name,
"filename": config_file_name,
"config": content
}
response = requests.post(self.add_config_url,
params=self.token_param,
json=request_data)
if TarsService.parse_response("add application config file",
response) is False:
return False
if response.status_code != 200:
return False
return True
def chain_operations(args):
if parser_handler.is_chain_command(args) is False:
return
if os.path.exists(args.config) is False:
utilities.log_error("The config file '%s' not found!" % args.config)
sys.exit(-1)
toml_config = toml.load(args.config)
chain_config = ChainConfig(toml_config)
op_type = args.type
if op_type not in ServiceInfo.supported_service_type:
utilities.log_error("the service type must be " +
', '.join(ServiceInfo.supported_service_type))
return
command = args.op
if op_type == ServiceInfo.rpc_service_type or op_type == ServiceInfo.gateway_service_type:
if command in CommandInfo.service_command_impl.keys():
command_impl = ServiceCommandImpl(chain_config, args.type)
impl_str = CommandInfo.service_command_impl[command]
cmd_func_attr = getattr(command_impl, impl_str)
cmd_func_attr()
return
if op_type == ServiceInfo.node_service_type:
if command in CommandInfo.node_command_to_impl.keys():
command_impl = NodeCommandImpl(chain_config)
impl_str = CommandInfo.node_command_to_impl[command]
cmd_func_attr = getattr(command_impl, impl_str)
cmd_func_attr()
return
utilities.log_info("unimplemented command")
def generate_ini_config(self):
"""
generate config.ini.tmp
"""
(_, generated_file_path) = self.get_ini_config_info()
if os.path.exists(generated_file_path) is True:
utilities.log_error(
"config file %s already exists, please delete after confirming carefully" % generated_file_path)
return False
ini_config = configparser.ConfigParser()
ini_config.read(self.tpl_config_path)
ini_config[self.section]['listen_ip'] = self.service_config.listen_ip
ini_config[self.section]['listen_port'] = str(
self.service_config.listen_port)
ini_config[self.section]['sm_ssl'] = utilities.convert_bool_to_str(
self.sm_ssl)
ini_config[self.section]['thread_count'] = str(
self.service_config.thread_count)
ini_config["service"]['gateway'] = self.config.chain_id + \
"." + self.service_config.gateway_service_name
ini_config["service"]['rpc'] = self.config.chain_id + \
"." + self.service_config.rpc_service_name
ini_config["chain"]['chain_id'] = self.config.chain_id
utilities.mkfiledir(generated_file_path)
with open(generated_file_path, 'w') as configfile:
ini_config.write(configfile)
utilities.log_info("* generate %s" % generated_file_path)
return True
def upgrade_all(self):
for service in self.service_dict.values():
if self.upgrade_service(service) is False:
utilities.log_error("upgrade service %s failed" % service.name)
return False
else:
utilities.log_info("upgrade service %s success" % service.name)
return True
def start_service(self, service_config):
for ip in service_config.deploy_ip:
tars_service = TarsService(self.config.tars_config.tars_url,
self.config.tars_config.tars_token,
self.config.chain_id, ip)
if tars_service.restart_server(service_config.name) is False:
utilities.log_error("start service for node %s failed" % ip)
return False
return True
def stop_all(self):
ret = True
for service in self.service_dict.values():
if self.stop_service(service) is False:
ret = False
utilities.log_error("stop service %s failed" % service.name)
else:
utilities.log_info("stop service %s success" % service.name)
return ret
def gen_service_config(self):
utilities.print_split_info()
utilities.print_badage("generate service config")
ret = self.service_controller.gen_all_service_config()
if ret is True:
utilities.print_badage("generate service config success")
else:
utilities.log_error("generate service config failed")
utilities.print_split_info()
return ret
def execute_command(self, function, notice_info):
utilities.print_split_info()
utilities.print_badage(notice_info)
ret = getattr(self.node_controller, function)()
if ret is True:
utilities.print_badage("%s success" % notice_info)
else:
utilities.log_error("%s failed" % notice_info)
utilities.print_split_info()
return ret
def upload_package(self, tars_service, service_name, org_service_name):
(ret, package_path) = utilities.try_to_rename_tgz_package(
"generated", self.config.tars_config.tars_pkg_dir, service_name,
org_service_name)
if ret is False:
utilities.log_error(
"upload package for service %s failed for rename package name failed"
% service_name)
return (False, -1)
return tars_service.upload_tars_package(service_name, package_path)
def gen_all_service_config(self):
for service in self.service_dict.values():
if self.gen_service_config(service) is False:
utilities.log_error("gen configuration for service %s failed" %
service.name)
return False
else:
utilities.log_info("gen configuration for service %s success" %
service.name)
return True
def get_server_info(self, tree_node_id):
params = {'tree_node_id': tree_node_id, "ticket": self.tars_token}
response = requests.get(self.get_server_list_url, params=params)
if TarsService.parse_response(
"get server info by tree node id: " + tree_node_id,
response) is False:
utilities.log_error(
"get server info by tree node id for error response, tree_node_id: %s, msg: %s"
% (tree_node_id, response.content))
return (False, response)
return (True, response)
def expand_server_with_preview(self, server_name, node_name,
expanded_node_list, obj_list):
ret = self.expand_server_preview(server_name, node_name,
expanded_node_list)
if ret is False:
utilities.log_error(
"expand server failed for expand preview failed, app: %s, server: %s, expanded_node_list: %s"
% (self.app_name, server_name, '.'.join(expanded_node_list)))
return False
return self.expand_server(server_name, node_name, expanded_node_list,
obj_list)
def stop_service(self, service_config):
for ip in service_config.deploy_ip:
tars_service = TarsService(self.config.tars_config.tars_url,
self.config.tars_config.tars_token,
self.config.chain_id, ip)
utilities.log_info("stop service %s, node: %s" %
(service_config.name, ip))
if tars_service.stop_server(service_config.name) is False:
utilities.log_error("stop service for node %s failed" % ip)
return False
return True
def undeploy_service(self, service_config):
for ip in service_config.deploy_ip:
tars_service = TarsService(self.config.tars_config.tars_url,
self.config.tars_config.tars_token,
self.config.chain_id, ip)
utilities.log_info("undeploy service for node %s, service: %s" %
(ip, service_config.name))
if tars_service.undeploy_tars(service_config.name) is False:
utilities.log_error("undeploy service %s for node %s failed" %
(ip, service_config.name))
return True
def get_docker_network_id(docker_network_name):
"""
get the docker network id
"""
command = "docker network ls | grep -i \"%s\" | awk -F\' \' \'{print $1}\'" % docker_network_name
(ret, result) = utilities.execute_command_and_getoutput(command)
if ret is False:
utilities.log_error("* get docker network id for %s failed" %
docker_network_name)
utilities.log_info("* get docker network id for %s success, id: %s" %
(docker_network_name, result))
return (ret, result)
def stop_service(self):
utilities.print_split_info()
utilities.print_badage("stop service, type: %s" % self.service_type)
ret = self.service_controller.stop_all()
if ret is True:
utilities.print_badage("stop service success, type: %s" %
self.service_type)
else:
utilities.log_error("stop service failed, type: %s" %
self.service_type)
utilities.print_split_info()
return ret
def deploy_service_list(self, service_list, obj_list, allow_duplicated):
"deploy service list"
i = 0
for service in service_list:
if self.deploy_single_service(service, obj_list[i],
allow_duplicated) is False:
utilities.log_error(
"deploy service list failed, service list: %s" %
service_list)
return False
i = i + 1
return True
def create_sub_net(subnet, docker_network_name):
"""
create the subnet
"""
utilities.log_info("* create docker subnet %s, name: %s" %
(subnet, docker_network_name))
command = "docker network create -d bridge --subnet=%s %s --opt com.docker.network.driver.mtu=1400" % (
subnet, docker_network_name)
if utilities.execute_command(command) is False:
utilities.log_error("create the docker subnet failed")
return False
return True
def add_node_config_list(self, config_list, service_name,
config_file_list):
i = 0
for config_file_path in config_file_list:
config = config_list[i]
if self.add_non_empty_server_config_file(
config, service_name, config_file_path) is False:
utilities.log_error(
"add_node_config_list failed, config files info: %s" %
config_list)
return False
i = i + 1
return True
def parse_response(operation, response):
if response.status_code != 200:
utilities.log_error(
"%s failed, error message: %s, error code: %d" %
(operation, response.content, response.status_code))
return False
result = response.json()
error_msg = result['err_msg']
if len(error_msg) > 0:
utilities.log_error("%s failed, error message: %s" %
(operation, error_msg))
return False
return True
def expand_all(self):
for service in self.service_dict.values():
if service.expanded_ip is None or len(service.expanded_ip) == 0:
utilities.log_error(
"Must set expanded_ip when expand service, service name: %s, deploy ip: %s, type: %s"
% (service, service.deploy_ip, self.service_type))
return False
if self.expand_service_list(service) is False:
utilities.log_error(
"expand service %s to %s failed, type: %s!" %
(service, service.deploy_ip, self.service_type))
return False
return True
def __init__(self, config):
self.config = config
section = "tars"
self.tars_url = utilities.get_value(self.config, section, "tars_url",
None, True)
self.tars_token = utilities.get_value(self.config, section,
"tars_token", None, True)
self.tars_pkg_dir = utilities.get_value(self.config, section,
"tars_pkg_dir", "binary",
False)
if len(self.tars_token) == 0:
utilities.log_error("Must config 'tars.tars_token'")
sys.exit(-1)
def upload_service(self):
# upload the generated_config
utilities.print_split_info()
utilities.print_badage("upload service, type: %s" % self.service_type)
ret = self.service_controller.deploy_all()
if ret is True:
utilities.print_badage("upload service success, type: %s" %
self.service_type)
else:
utilities.log_error("upload service failed, type: %s" %
self.service_type)
utilities.print_split_info()
return ret