def expand_server_preview(self, server_name, node_name, expanded_node_list): """ expand the server preview """ utilities.log_info( "expand_server_preview, app: %s, server_name: %s, expanded_node_list: %s" % (self.app_name, server_name, '.'.join(expanded_node_list))) request_data = { "application": self.app_name, "server_name": server_name, "node_name": node_name, "set": "", "expand_nodes": expanded_node_list, "enable_set": "false", "set_name": "", "set_area": "", "set_group": "", "copy_node_config": "false", "nodeName": [] } response = requests.post(self.expand_server_preview_url, params=self.token_param, json=request_data) if TarsService.parse_response("expand server preview", response) is False: utilities.log_error( "expand server preview for error response, server name: %s, msg: %s" % (server_name, response.content)) return False utilities.log_info("expand server preview response %s" % response.content) return True
def chain_operations(args): if parser_handler.is_chain_command(args) is False: return if os.path.exists(args.config) is False: utilities.log_error("The config file '%s' not found!" % args.config) sys.exit(-1) toml_config = toml.load(args.config) chain_config = ChainConfig(toml_config) op_type = args.type if op_type not in ServiceInfo.supported_service_type: utilities.log_error("the service type must be " + ', '.join(ServiceInfo.supported_service_type)) return command = args.op if op_type == ServiceInfo.rpc_service_type or op_type == ServiceInfo.gateway_service_type: if command in CommandInfo.service_command_impl.keys(): command_impl = ServiceCommandImpl(chain_config, args.type) impl_str = CommandInfo.service_command_impl[command] cmd_func_attr = getattr(command_impl, impl_str) cmd_func_attr() return if op_type == ServiceInfo.node_service_type: if command in CommandInfo.node_command_to_impl.keys(): command_impl = NodeCommandImpl(chain_config) impl_str = CommandInfo.node_command_to_impl[command] cmd_func_attr = getattr(command_impl, impl_str) cmd_func_attr() return utilities.log_info("unimplemented command")
def create_bridge(docker_network_name, docker_vxlan_name, remote_ip): """ add the bridge """ dstport = 4789 dev_name = "eth0" utilities.log_info( "* set the bridge interconnection network, docker_network: %s, docker_vxlan_name: %s, remote_ip: %s, dstport: %s" % (docker_network_name, docker_vxlan_name, remote_ip, dstport)) (ret, network_id ) = NetworkManager.get_docker_network_id(docker_network_name) basic_error_info = "Failed to set the bridge interconnection network" if ret is False: utilities.log_error( "%s, please check the network name! remote ip: %s, network name: %s" % (basic_error_info, remote_ip, docker_network_name)) return False # add ip link ip_link_command = "ip link add %s type vxlan id 200 remote %s dstport %d dev %s" % ( docker_vxlan_name, remote_ip, dstport, dev_name) if utilities.execute_command(ip_link_command) is False: utilities.log_error("%s" % basic_error_info) return False # setup the network ip_set_up_command = "ip link set %s up" % docker_vxlan_name if utilities.execute_command(ip_set_up_command) is False: utilities.log_error("%s" % basic_error_info) return False # add bridge bridge_add_command = "brctl addif br-%s %s" % (network_id, docker_vxlan_name) if utilities.execute_command(bridge_add_command) is False: utilities.log_error("%s" % basic_error_info) return False return True
def generate_ini_config(self): """ generate config.ini.tmp """ (_, generated_file_path) = self.get_ini_config_info() if os.path.exists(generated_file_path) is True: utilities.log_error( "config file %s already exists, please delete after confirming carefully" % generated_file_path) return False ini_config = configparser.ConfigParser() ini_config.read(self.tpl_config_path) ini_config[self.section]['listen_ip'] = self.service_config.listen_ip ini_config[self.section]['listen_port'] = str( self.service_config.listen_port) ini_config[self.section]['sm_ssl'] = utilities.convert_bool_to_str( self.sm_ssl) ini_config[self.section]['thread_count'] = str( self.service_config.thread_count) ini_config["service"]['gateway'] = self.config.chain_id + \ "." + self.service_config.gateway_service_name ini_config["service"]['rpc'] = self.config.chain_id + \ "." + self.service_config.rpc_service_name ini_config["chain"]['chain_id'] = self.config.chain_id utilities.mkfiledir(generated_file_path) with open(generated_file_path, 'w') as configfile: ini_config.write(configfile) utilities.log_info("* generate %s" % generated_file_path) return True
def expand_service_list(self, service_config): for ip in service_config.deploy_ip: utilities.log_info("expand to %s, app: %s, name: %s" % (ip, self.config.chain_id, service_config.name)) if self.expand_service_to_given_ip( service_config, service_config.expanded_ip, ip) is False: return False return True
def upgrade_all(self): for service in self.service_dict.values(): if self.upgrade_service(service) is False: utilities.log_error("upgrade service %s failed" % service.name) return False else: utilities.log_info("upgrade service %s success" % service.name) return True
def generate_rpc_config_files(self): utilities.log_info("* generate config for the rpc service") if self.generate_ini_config() is False: return False if self.generate_cert() is False: return False utilities.log_info("* generate config for the rpc service success") return True
def upgrade_service(self, service_config): for ip in service_config.deploy_ip: utilities.log_info("upgrade_service %s to %s" % (service_config.name, ip)) ret = self.upgrade_service_to_given_ip(service_config, ip) if ret is False: return False return True
def stop_all(self): ret = True for service in self.service_dict.values(): if self.stop_service(service) is False: ret = False utilities.log_error("stop service %s failed" % service.name) else: utilities.log_info("stop service %s success" % service.name) return ret
def gen_all_service_config(self): for service in self.service_dict.values(): if self.gen_service_config(service) is False: utilities.log_error("gen configuration for service %s failed" % service.name) return False else: utilities.log_info("gen configuration for service %s success" % service.name) return True
def generate_gateway_config_files(self): utilities.log_info("* generate config for the gateway service") if self.generate_ini_config() is False: return False if self.generate_cert() is False: return False if self.generate_gateway_connection_info() is False: return False utilities.log_info("* generate config for the gateway service success") return True
def undeploy_service(self, service_config): for ip in service_config.deploy_ip: tars_service = TarsService(self.config.tars_config.tars_url, self.config.tars_config.tars_token, self.config.chain_id, ip) utilities.log_info("undeploy service for node %s, service: %s" % (ip, service_config.name)) if tars_service.undeploy_tars(service_config.name) is False: utilities.log_error("undeploy service %s for node %s failed" % (ip, service_config.name)) return True
def gen_service_config(self, service_config): for ip in service_config.deploy_ip: utilities.log_info("* generate service config for %s : %s" % (ip, service_config.name)) config_generator = ServiceConfigGenerator(self.config, self.service_type, service_config, ip) ret = config_generator.generate_all_config() if ret is False: return ret return True
def stop_service(self, service_config): for ip in service_config.deploy_ip: tars_service = TarsService(self.config.tars_config.tars_url, self.config.tars_config.tars_token, self.config.chain_id, ip) utilities.log_info("stop service %s, node: %s" % (service_config.name, ip)) if tars_service.stop_server(service_config.name) is False: utilities.log_error("stop service for node %s failed" % ip) return False return True
def get_docker_network_id(docker_network_name): """ get the docker network id """ command = "docker network ls | grep -i \"%s\" | awk -F\' \' \'{print $1}\'" % docker_network_name (ret, result) = utilities.execute_command_and_getoutput(command) if ret is False: utilities.log_error("* get docker network id for %s failed" % docker_network_name) utilities.log_info("* get docker network id for %s success, id: %s" % (docker_network_name, result)) return (ret, result)
def create_sub_net(subnet, docker_network_name): """ create the subnet """ utilities.log_info("* create docker subnet %s, name: %s" % (subnet, docker_network_name)) command = "docker network create -d bridge --subnet=%s %s --opt com.docker.network.driver.mtu=1400" % ( subnet, docker_network_name) if utilities.execute_command(command) is False: utilities.log_error("create the docker subnet failed") return False return True
def generate_cert(self): output_dir = self.get_cert_output_dir() if self.ca_generated() is False: # generate the ca cert utilities.generate_ca_cert(self.sm_ssl, self.ca_cert_path) utilities.log_info("* generate cert, output path: %s" % (output_dir)) utilities.generate_node_cert( self.sm_ssl, self.ca_cert_path, output_dir) if self.service_type == ServiceInfo.rpc_service_type: utilities.log_info( "* generate sdk cert, output path: %s" % (output_dir)) utilities.generate_sdk_cert( self.sm_ssl, self.ca_cert_path, output_dir) return True
def generate_gateway_connection_info(self): (_, generated_file_path) = self.get_network_connection_config_info() if os.path.exists(generated_file_path): utilities.log_error( "config file %s already exists, please delete after confirming carefully" % generated_file_path) return False peers = {} peers["nodes"] = self.service_config.peers utilities.mkfiledir(generated_file_path) with open(generated_file_path, 'w') as configfile: json.dump(peers, configfile) utilities.log_info( "* generate gateway connection file: %s" % generated_file_path) return True
def download_binary(self, binary_name, download_url): if os.path.exists(self.binary_path) is False: utilities.mkdir(self.binary_path) binary_file_path = self.get_required_binary_path(binary_name) utilities.log_info("Download url: %s" % download_url) with open(binary_file_path, 'wb') as file: response = requests.get(download_url, stream=True) total = response.headers.get('content-length') if total is None or int(total) < 100000: utilities.log_error( "Download binary %s failed, Please check the existence of the binary version %s" % (binary_name, self.version)) return False utilities.log_info( "* Download %s from %s\n* size: %fMB, dst_path: %s" % (binary_name, download_url, float(total) / float(1000000), binary_file_path)) downloaded = 0 total = int(total) for data in response.iter_content( chunk_size=max(int(total / 1000), 1024 * 1024)): downloaded += len(data) file.write(data) done = int(50 * downloaded / total) utilities.log_info("Download percent: %d%%" % (downloaded / total * 100)) sys.stdout.write('\r[{}{}]'.format('█' * done, '.' * (50 - done))) sys.stdout.flush() sys.stdout.write('\n') utilities.log_info("* Download %s from %s success" % (binary_name, download_url)) return True
def expand_server(self, server_name, node_name, expanded_node_list, obj_list): """ expand the server """ utilities.log_info("expand_server, app: %s, server_name: %s" % (self.app_name, server_name)) expand_servers_info = [] for node in expanded_node_list: for obj in obj_list: (ret, port) = self.get_node_auto_port(node) if ret is False: utilities.log_error( "expand server failed for get node auto port failed, server: %s, node: %s" % (server_name, node)) return False node_info = { "bind_ip": node, "node_name": node, "obj_name": obj, "port": port, "set": "" } expand_servers_info.append(node_info) request_data = { "application": self.app_name, "server_name": server_name, "set": "", "node_name": node_name, "copy_node_config": "false", "expand_preview_servers": expand_servers_info } response = requests.post(self.expand_server_url, params=self.token_param, json=request_data) if TarsService.parse_response("expand server", response) is False: utilities.log_error( "expand server for error response, server name: %s, msg: %s" % (server_name, response.content)) return False utilities.log_info("expand server response %s" % response.content) return True
def deploy_single_service(self, service_name, obj_name_list, allow_duplicated): "deploy single service" if self.server_exists( service_name) is True and allow_duplicated is False: utilities.log_error("service %s already exists." % service_name) return False utilities.log_info("deploy service %s" % service_name) adapters = [] for obj_name in obj_name_list: # get the un-occupied port (ret, port) = self.get_auto_port() if ret is False: utilities.log_error( "deploy service %s failed for get un-occupied port failed" % service_name) return False adapters.append({ "obj_name": obj_name, "port": port, "bind_ip": self.deploy_ip, "port_type": "tcp", "thread_num": 5, "max_connections": 100000, "queuecap": 50000, "queuetimeout": 20000 }) request_data = { "application": self.app_name, "server_name": service_name, "node_name": self.deploy_ip, "server_type": "tars_cpp", "template_name": "tars.cpp.default", 'adapters': adapters } response = requests.post(self.deploy_service_url, params=self.token_param, json=request_data) if TarsService.parse_response("deploy service " + service_name, response) is False: return False return True
def upload_tars_package(self, service_name, package_path): """ upload the tars package """ package_name = service_name + ServiceInfo.tars_pkg_postfix utilities.log_debug( "upload tars package for service %s, package_path: %s, package_name: %s" % (service_name, package_path, package_name)) if os.path.exists(package_path) is False: utilities.log_error( "upload tars package for service %s failed for the path %s not exists" % (service_name, package_path)) return (False, 0) task_id = str(uuid.uuid4()) form_data = MultipartEncoder( fields={ "application": self.app_name, "task_id": task_id, "module_name": service_name, "comment": "upload package", "suse": (package_name, open(package_path, 'rb'), 'text/plain/binary') }) response = requests.post( self.upload_package_url, data=form_data, params=self.token_param, headers={'Content-Type': form_data.content_type}) if TarsService.parse_response("upload tars package " + package_path, response) is False: return (False, 0) # get the id (ret, id) = self.get_server_patch(task_id) if ret is True: utilities.log_info("upload tar package %s success, config id: %s" % (package_path, id)) return (ret, id)
def add_config_list(self, config_list, service_name, node_name, config_file_list, empty_server_config): i = 0 for config_file_path in config_file_list: config = config_list[i] utilities.log_info( "* add config for service %s, node: %s, config: %s" % (service_name, node_name, config)) utilities.log_debug( "add config for service %s, node: %s, config: %s, path: %s" % (service_name, node_name, config, config_file_path)) if self.add_config_file(config, service_name, node_name, config_file_path, empty_server_config) is False: utilities.log_error( "add_config_list failed, config files info: %s" % config_list) return False i = i + 1 return True
def deploy_service(self, service_config): if len(service_config.deploy_ip) == 0: utilities.log_info("No service to deploy") deploy_ip = (service_config.deploy_ip)[0] utilities.log_info( "deploy_service to %s, app: %s, name: %s" % (deploy_ip, self.config.chain_id, service_config.name)) self.deploy_service_to_given_ip(service_config, deploy_ip) for i in range(1, len(service_config.deploy_ip)): ip = service_config.deploy_ip[i] utilities.log_info("expand service to %s, app: %s, name: %s" % (ip, self.config.chain_id, service_config.name)) if self.expand_service_to_given_ip(service_config, deploy_ip, ip) is False: return False return True