def check(self, data_node_info_list):
confOpers = ConfigFileOpers()
false_nodes, value, _password = [], {}, ''
value = confOpers.getValue(
options.mysql_cnf_file_name)["wsrep_sst_auth"]
_password = value.split(":")[1][:-1]
for data_node_ip in data_node_info_list:
conn = self.dba_opers.get_mysql_connection(data_node_ip,
user="******",
passwd=_password)
if conn == None:
false_nodes.append(data_node_ip)
else:
try:
rows = self.dba_opers.show_status(conn)
finally:
conn.close()
key_value = retrieve_kv_from_db_rows(
rows, ['wsrep_incoming_addresses', 'wsrep_cluster_size'])
node_size_dict = self._check_wsrep_incoming_addresses(
key_value, data_node_info_list)
return node_size_dict
if (len(false_nodes) == 3):
exception_dict = {}
exception_dict.setdefault("message", "no way to connect to db")
exception_dict.setdefault("alarm", options.alarm_serious)
return exception_dict
class AdminUser(APIHandler):
confOpers = ConfigFileOpers()
def post(self):
requestParam = {}
args = self.request.arguments
logging.info("args :" + str(args))
if not args:
raise HTTPAPIErrorException("params is empty")
for key in args:
value = args[key][0]
if key == 'adminPassword':
value = base64.encodestring(value).strip('\n')
requestParam.setdefault(key, value)
if "adminUser" not in requestParam or 'adminPassword' not in requestParam:
raise HTTPAPIErrorException(
"admin user or password is empty, please check it!")
self.confOpers.setValue(options.cluster_property, requestParam)
result = {}
# dict.setdefault("code", '000000')
result.setdefault("message", "creating admin user successful!")
self.finish(result)
class Stopgbalancer(APIHandler):
confOpers = ConfigFileOpers()
invokeCommand = InvokeCommand()
def post(self):
port = self.get_argument('port', None)
if port is None or port == '':
raise HTTPAPIError(status_code=417, error_detail="lost port argument",\
notification = "direct", \
log_message= "start GLB: lost port argument",\
response = "please input port.")
cmd = r'netstat -ntlp|grep -w %s|grep -v grep' % (port)
result = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
if result[0] == '':
raise HTTPAPIError(status_code=417, error_detail="this glb is stopped",\
notification = "direct", \
log_message= "this glb is stopped", \
response = "please check envirment")
logging.info("stop gbalancer(port): %s" % (port))
temp = re.split('\s+', result[0])
glb_pid = temp[len(temp) - 1].split('/')[0]
self.invokeCommand.run_check_shell(r'kill -9 %s' % (glb_pid))
glb_proc = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
if not glb_proc[0] == '':
raise HTTPAPIError(status_code=417, error_detail="cannot stop glb",\
notification = "direct", \
log_message= "cannot stop glb", \
response = "please check envirment")
dict = {}
#dict.setdefault("code", '000000')
dict.setdefault("message", "stop gbalancer successful!")
self.finish(dict)
class AdminConf(APIHandler):
confOpers = ConfigFileOpers()
adminOpers = AdminOpers()
def post(self):
requestParam = {}
args = self.request.arguments
for key in args:
requestParam.setdefault(key, args[key][0])
if not requestParam:
raise HTTPAPIErrorException("params is empty")
#TODO:
'''1.judge throwing code reuse rate is very low.
2.thrown error code directly, automatically reads the type of error, error messages, and so on.'''
if "zkAddress" not in requestParam or 'zkPort' not in requestParam:
raise HTTPAPIErrorException(
"zkaddress or port is empty, please check it!")
if self.confOpers.ipFormatChk(requestParam['zkAddress']):
raise HTTPAPIErrorException("zkaddress is illegal",
status_code=417)
self.confOpers.setValue(options.mcluster_manager_cnf, requestParam)
self.adminOpers.sync_info_from_zk(requestParam['zkAddress'][0])
result = {}
# dict.setdefault("code", '000000')
result.setdefault("message", "admin conf successful!")
self.finish(result)
class InitMCluster(APIHandler):
confOpers = ConfigFileOpers()
invokeCommand = InvokeCommand()
def get(self):
args = self.request.arguments
forceInit = args['forceInit'][0]
isLock = False
lock = None
try:
zkOper = self.retrieve_zkOper()
#existCluster = zkOper.existCluster()
isLock,lock = zkOper.lock_init_node_action()
dataNodeProKeyValue = self.confOpers.getValue(options.data_node_property, ['dataNodeIp','dataNodeName'])
data_node_ip = dataNodeProKeyValue['dataNodeIp']
data_node_name = dataNodeProKeyValue['dataNodeName']
clusterProKeyValue = self.confOpers.getValue(options.cluster_property, ['clusterUUID','clusterName'])
clusterUUID = clusterProKeyValue['clusterUUID']
#check if cluster has odd data node
if not forceInit:
dataNodeNumber = zkOper.getDataNodeNumber(clusterUUID)
if dataNodeNumber / 2 == 0:
raise HTTPAPIErrorException("the number should be not odd number,please add 1 or 3 data node into cluster!",
status_code=417)
clusterName = clusterProKeyValue['clusterName']
clusterAddress = 'gcomm://%s' % (data_node_ip)
requestParam = {'wsrep_cluster_name': clusterName, 'wsrep_node_address': data_node_ip, 'wsrep_cluster_address':clusterAddress, 'wsrep_node_name':data_node_name}
self.confOpers.setValue(options.mysql_cnf_file_name, requestParam)
sst_user_password = self.invokeCommand.runBootstrapScript()
mysql_cnf_text = self.confOpers.retrieveFullText(options.mysql_cnf_file_name)
zkOper.writeMysqlCnf(clusterUUID, mysql_cnf_text)
zkOper.write_started_node(data_node_ip)
except kazoo.exceptions.LockTimeout:
raise HTTPAPIErrorException("the mysql cluster is initing,please wait for the completion of other machine join this cluster.",
status_code=417)
finally:
if isLock:
zkOper.unLock_init_node_action(lock)
result = {}
# dict.setdefault("code", '000000')
result.setdefault("message", "init cluster successful!")
result.setdefault("sst_user_password", sst_user_password)
self.finish(result)
class CreateMCluster(APIHandler):
confOpers = ConfigFileOpers()
def post(self):
#check if exist cluster
zkOper = self.retrieve_zkOper()
existCluster = zkOper.existCluster()
if existCluster:
if zkOper.judgeClusterStatus("remove"):
zkOper.remove_zk_info(CLUSTER_NAME)
else:
raise HTTPAPIErrorException("server has belong to a cluster,should be not create new cluster!", status_code=417)
requestParam = {}
args = self.request.arguments
logging.info("args :" + str(args))
if not args:
raise HTTPAPIErrorException("params is empty")
for key in args:
value = args[key][0]
requestParam.setdefault(key, value)
if "clusterName" not in requestParam or 'dataNodeName' not in requestParam:
raise HTTPAPIErrorException("cluster_name or node_name is empty, please check it!")
if 'dataNodeIp' not in requestParam:
raise HTTPAPIErrorException("node_ip is empty, please check it!")
cluster_name = requestParam['clusterName']
if len(cluster_name) >= 33:
raise HTTPAPIErrorException("Cluster name is too long, please use name whoes length is less than 33 characters",
status_code=417)
if self.confOpers.ipFormatChk(requestParam['dataNodeIp']):
raise HTTPAPIErrorException("dataNodeIp is illegal", status_code=417)
clusterUUID = cluster_name + '/' + str(uuid.uuid1())
requestParam.setdefault("clusterUUID",clusterUUID)
if requestParam != {}:
self.confOpers.setValue(options.cluster_property, requestParam)
self.confOpers.setValue(options.data_node_property, requestParam)
clusterProps = self.confOpers.getValue(options.cluster_property)
dataNodeProprs = self.confOpers.getValue(options.data_node_property)
zkOper.writeClusterInfo(clusterUUID, clusterProps)
zkOper.writeDataNodeInfo(clusterUUID, dataNodeProprs)
result = {}
# dict.setdefault("code", '000000')
result.setdefault("message", "creating cluster successful!")
self.finish(result)
def sync_info_from_zk(self, node_ip_addr):
zkOper = ZkOpers()
try:
cluster_existed = zkOper.existCluster()
if cluster_existed:
clusterUUID = zkOper.getClusterUUID()
data, _ = zkOper.retrieveClusterProp(clusterUUID)
node_ip_addr = get_localhost_ip()
assert node_ip_addr
return_result = zkOper.retrieve_data_node_info(node_ip_addr)
json_str_data = data.replace("'", "\"")
dict_data = json.loads(json_str_data)
if type(return_result) is dict and type(dict_data) is dict:
config_file_obj = ConfigFileOpers()
config_file_obj.setValue(options.data_node_property,
return_result)
config_file_obj.setValue(options.cluster_property,
dict_data)
logging.debug(
"program has re-written zk data into configuration file"
)
else:
logging.info("write data into configuration failed")
finally:
zkOper.stop()
class DBCreate(APIHandler):
dba_opers = DBAOpers()
conf_opers = ConfigFileOpers()
def post(self):
dbName = self.get_argument("dbName", None)
userName = self.get_argument("userName", None)
ip_address = self.get_argument("ip_address", '%')
max_queries_per_hour = self.get_argument("max_queries_per_hour", 0)
max_updates_per_hour = self.get_argument("max_updates_per_hour", 0)
max_connections_per_hour = self.get_argument(
"max_connections_per_hour", 0)
max_user_connections = self.get_argument("max_user_connections", 200)
userPassword = get_random_password()
conn = self.dba_opers.get_mysql_connection()
try:
self.dba_opers.craete_database(conn, dbName)
self.dba_opers.create_user(conn, userName, userPassword,
ip_address)
self.dba_opers.grant_manager_privileges(
conn, userName, userPassword, dbName, ip_address,
max_queries_per_hour, max_updates_per_hour,
max_connections_per_hour, max_user_connections)
self.dba_opers.flush_privileges(conn)
finally:
conn.close()
# check if exist cluster
dbProps = {'db_name': dbName}
zkOper = self.retrieve_zkOper()
clusterUUID = zkOper.getClusterUUID()
zkOper.write_db_info(clusterUUID, dbName, dbProps)
userProps = {
'role': 'manager',
'max_queries_per_hour': max_queries_per_hour,
'max_updates_per_hour': max_updates_per_hour,
'max_connections_per_hour': max_connections_per_hour,
'max_user_connections': max_user_connections
}
zkOper.write_user_info(clusterUUID, dbName, userName, ip_address,
userProps)
result = {}
result.setdefault("message", "database create successful")
result.setdefault("manager_user_name", userName)
result.setdefault("manager_user_password", userPassword)
self.finish(result)
class SyncDataNode(APIHandler):
confOpers = ConfigFileOpers()
def get(self, ip_address):
if ip_address is None:
error_message = "you should specify the ip address need to sync"
raise HTTPAPIErrorException(error_message, status_code=417)
zkOper = self.retrieve_zkOper()
return_result = zkOper.retrieve_data_node_info(ip_address)
self.confOpers.setValue(options.data_node_property, return_result)
result = {}
# dict.setdefault("code", "000000")
result.setdefault("message", "sync data node info to local successful!")
self.finish(result)
class SyncMCluster(APIHandler):
confOpers = ConfigFileOpers()
def get(self):
zkOper = self.retrieve_zkOper()
existCluster = zkOper.existCluster()
clusterUUID = zkOper.getClusterUUID()
data, _ = zkOper.retrieveClusterProp(clusterUUID)
self.confOpers.setValue(options.cluster_property, eval(data))
result = {}
# dict.setdefault("code", '000000')
result.setdefault("message", "sync mcluster info to local successful!")
self.finish(result)
class Check_Cluster_Available(Check_Status_Base):
dba_opers = DBAOpers()
confOpers = ConfigFileOpers()
def check(self, data_node_info_list):
success_nodes, value, _password = [], {}, ''
value = self.confOpers.getValue(
options.mysql_cnf_file_name)["wsrep_sst_auth"]
_password = value.split(":")[1][:-1]
for data_node_ip in data_node_info_list:
try:
conn = self.dba_opers.get_mysql_connection(data_node_ip,
user="******",
passwd=_password)
if conn is not None:
success_nodes.append(data_node_ip)
finally:
if conn is not None:
conn.close()
message = "no avaliable data node"
if len(success_nodes) >= 1:
message = 'ok'
total_count = len(data_node_info_list)
success_count = len(success_nodes)
failed_count = total_count - success_count
alarm_result = self.retrieve_alarm_level(total_count, success_count,
failed_count)
cluster_available_dict = {}
cluster_available_dict.setdefault("message", message)
cluster_available_dict.setdefault("alarm", alarm_result)
return cluster_available_dict
def retrieve_alarm_level(self, total_count, success_count, failed_count):
if success_count == 0:
return options.alarm_serious
return options.alarm_nothing
class DBDelete(APIHandler):
dba_opers = DBAOpers()
conf_opers = ConfigFileOpers()
def delete(self, dbName):
if not dbName:
raise HTTPAPIErrorException(
"when remove the db, no have database name,\
please provide database name you want to removed!",
status_code=417)
zkOper = self.retrieve_zkOper()
clusterUUID = zkOper.getClusterUUID()
user_ipAddress_map = zkOper.retrieve_db_user_prop(clusterUUID, dbName)
conn = self.dba_opers.get_mysql_connection()
try:
if user_ipAddress_map is not None:
for (user_name, ip_address) in user_ipAddress_map.items():
self.dba_opers.delete_user(conn, user_name, ip_address)
self.dba_opers.drop_database(conn, dbName)
finally:
conn.close()
user_name_list = ''
if user_ipAddress_map is not None:
for (user_name, ip_address) in user_ipAddress_map.items():
zkOper.remove_db_user(clusterUUID, dbName, user_name,
ip_address)
user_name_list += user_name + ","
zkOper.remove_db(clusterUUID, dbName)
result = {}
result.setdefault("message", "database remove successful!")
result.setdefault("removed_db_name", dbName)
result.setdefault("removed_user_with_db_name", user_name_list)
self.finish(result)
class Arbitrator(object):
'''
@todo: Arbitrator class need to review
'''
confOpers = ConfigFileOpers()
def __init__(self):
'''
Constructor
'''
def communicate(self, peer_ip, url):
http_client = tornado.httpclient.HTTPClient()
requesturi = "http://" + peer_ip + ":" + str(options.port) + url
try:
response = http_client.fetch(requesturi)
except tornado.httpclient.HTTPError as e:
logging.error(str(e))
http_client.close()
return "error"
logging.info(str(response.body))
return response.body
def get_ip(self, data_node_ip_list):
ret_dict = self.confOpers.getValue(options.data_node_property,
['dataNodeName', 'dataNodeIp'])
node_name = ret_dict['dataNodeName']
obj = re.search("-n-2", node_name)
if obj != None:
return ret_dict['dataNodeIp']
else:
tem_ip_list = data_node_ip_list
tem_ip_list.remove(ret_dict['dataNodeIp'])
result = ""
for ip in tem_ip_list:
url_post = "/inner/arbitrator/ip"
result = self.communicate(ip, url_post)
if result != "false":
break
return result
class Node_Mysql_Service_Opers(Abstract_Mysql_Service_Opers):
invokeCommand = InvokeCommand()
confFileOper = ConfigFileOpers()
def __init__(self):
'''
Constructor
'''
def retrieve_recover_position(self):
result = self.invokeCommand.run_check_shell(
options.retrieve_node_uuid_seqno_script)
uuid = self.__find_special_value(result, "uuid:", 37)
seqno = self.__find_special_value(result, "seqno:", 65535)
result = {}
result.setdefault("uuid", uuid)
result.setdefault("seqno", seqno)
return result
def __find_special_value(self, result, key, value_length):
key_start_pos = result.find(key)
key_end_pos = key_start_pos + len(key)
value = result[key_end_pos:key_end_pos + value_length]
value = value.rstrip('\n')
return value
def start(self, isNewCluster):
'''
@todo: check only pass lock to below action and close the zkOper object, real action if can release the lock
'''
node_start_action = Node_start_action(isNewCluster)
node_start_action.start_run()
def stop(self):
# Stop a thread to run the events
node_stop_action = Node_stop_action()
node_stop_action.start()
def retrieve_log_for_error(self):
# result = self.invokeCommand.run_check_shell(options.check_datanode_error)
result = 'true'
# if cmp('false',result) == 0:
#_tmp_error_log_file_path = '/tmp_check_datanode_error'
#_mysql_error_log_message = self.confFileOper.retrieveFullText(_tmp_error_log_file_path)
#_email_subject = "[%s] MySQL log error message" % options.sitename
#self._send_log_info_email(_email_subject, _mysql_error_log_message)
return result
def retrieve_log_for_warning(self):
# result = self.invokeCommand.run_check_shell(options.check_datanode_warning)
result = 'true'
# if cmp('false',result) == 0:
#_tmp_warning_log_file_path = '/tmp_check_datanode_warning'
#_mysql_error_log_message = self.confFileOper.retrieveFullText(_tmp_warning_log_file_path)
#_email_subject = "[%s] MySQL log warning message" % options.sitename
#self._send_log_info_email(_email_subject, _mysql_error_log_message)
return result
def _send_log_info_email(self, subject, content):
local_ip = get_localhost_ip()
# send email
# body = self.render_string("errors/500_email.html", exception=content)
body = content + "\nip:" + local_ip
if options.send_email_switch:
send_email(options.admins, subject, body)
class Abstract_Stat_Service(object):
invokeCommand = InvokeCommand()
confOpers = ConfigFileOpers()
zkOper = None
def __init__(self):
'''
Constructor
'''
def retrieve_zkOper(self):
if None == self.zkOper:
self.zkOper = Abstract_ZkOpers()
return self.zkOper
@abstractmethod
def stat(self):
raise NotImplementedError, "Cannot call abstract method"
def _retrieve_dict_with_result(self, stat_command):
return_result = self.invokeCommand.run_check_shell(stat_command)
title_value_result = return_result.split('\n')
title_list = title_value_result[0].split('\t')
value_list = []
if len(title_value_result) == 2:
value_list = title_value_result[1].split('\t')
result = {}
for i in range(len(title_list)):
title = title_list[i]
if value_list == [] or value_list is None:
value = 0
elif value_list[
0] == "127.0.0.1: Can't connect to MySQL server on '127.0.0.1' (111)":
value = 0
elif value_list[0] == '':
value = 0
else:
value = value_list[i]
result.setdefault(title, value)
return result
def _convert_dict_to_str(self, param):
title_list_str = ''
value_list_str = ''
for (key, value) in param.iteritems():
title_list_str += key + '\t'
value_list_str += value + '\t'
title_list_str = title_list_str.rstrip('\t')
value_list_str = value_list_str.rstrip('\t')
return_result = """{title_list_str}\n{value_list_str}""".format(
title_list_str=title_list_str, value_list_str=value_list_str)
return return_result
def _split_key_value(self, key_list, target_dict):
sub_dict = {}
for i in range(len(key_list)):
title = key_list[i]
value = target_dict.get(title)
sub_dict.setdefault(title, value)
return sub_dict
def _check_mysql_processor_exist(self):
zkOper = self.retrieve_zkOper()
try:
started_nodes = zkOper.retrieve_started_nodes()
except:
started_nodes = []
confDict = self.confOpers.getValue(options.data_node_property,
['dataNodeIp'])
data_node_ip = confDict['dataNodeIp']
processor_existed = False
for started_node in started_nodes:
if started_node == data_node_ip:
processor_existed = True
break
return processor_existed
class Inner_DB_Check_WR(APIHandler):
dba_opers = DBAOpers()
confOpers = ConfigFileOpers()
invokeCommand = InvokeCommand()
#
def get(self):
conn = self.dba_opers.get_mysql_connection()
try:
dataNodeProKeyValue = self.confOpers.getValue(
options.data_node_property, ['dataNodeIp'])
data_node_ip = dataNodeProKeyValue['dataNodeIp']
zkOper = self.retrieve_zkOper()
started_ip_list = zkOper.retrieve_started_nodes()
identifier = socket.gethostname()
'''
@todo: review the comment code for arbitrator way
'''
# ret_dict = self.confOpers.getValue(options.data_node_property, ['dataNodeName','dataNodeIp'])
# node_name = ret_dict['dataNodeName']
# obj = re.search("-n-2", node_name)
# if obj != None:
# self.finish("true")
# return
if conn is None:
if data_node_ip in started_ip_list:
zkOper.remove_started_node(data_node_ip)
self.invokeCommand.run_check_shell(options.kill_innotop)
self.finish("false")
return
zkOper.write_started_node(data_node_ip)
if not is_monitoring(get_localhost_ip(), zkOper):
self.finish("true")
return
dbName = 'monitor'
n_time = datetime.datetime.now()
h = n_time.hour
min = n_time.minute
offset = h / 6
tbName = ''
prefix_tb_name = 'tem'
mid_tb_name = str(identifier)
mid_tb_name_rps = mid_tb_name.replace("-", "_")
pre_tbname = prefix_tb_name + mid_tb_name_rps
for i in range(4):
tbName = pre_tbname + "_" + str(i)
self.dba_opers.check_create_table(conn, tbName, dbName)
tbName = pre_tbname + "_" + str(offset)
del_tbName = ''
ft = float(time.time())
if h % 6 == 0 and min <= 59 and (1000000 * ft) % 10 == 0:
int_tbName = (offset + 2) % 4
del_tbName = "%s_%s" % (pre_tbname, int_tbName)
self.dba_opers.delete_tb_contents(conn, del_tbName, dbName)
logging.info(
'delete the contents in database (%s) before 12 hours success!'
% (del_tbName))
str_time = n_time.strftime(TIME_FORMAT)
self.dba_opers.insert_record_time(conn, str_time, identifier,
tbName, dbName)
logging.info('Insert time %s into table %s ' % (str_time, tbName))
record_time = self.dba_opers.query_record_time(
conn, identifier, tbName, dbName)
except Exception, e:
return_flag = 'false'
logging.error(e)
self.finish(return_flag)
return
finally:
class Startgbalancer(APIHandler):
confOpers = ConfigFileOpers()
invokeCommand = InvokeCommand()
def post(self):
user = self.get_argument('user', None)
passwd = self.get_argument('passwd', None)
service = self.get_argument('service', None)
iplist_port = self.get_argument('iplist_port', None)
port = self.get_argument('port', None)
args = self.get_argument('args', None)
if service is None:
raise HTTPAPIError(status_code=417, error_detail="lost service arguments", \
notification = "direct", \
log_message= "start GLB: lost service argument",\
response = "please input service.")
if service == 'mysql':
lost_args = []
if user is None:
lost_args.append("user")
if passwd is None:
lost_args.append("passwd")
if len(lost_args) != 0:
raise HTTPAPIError(status_code=417, error_detail="lost %s arguments" % (lost_args), \
notification = "direct", \
log_message= "start GLB: lost %s argument" % (lost_args),\
response = "please input %s." % (lost_args))
if iplist_port is None or iplist_port == '':
raise HTTPAPIError(status_code=417, error_detail="lost iplist_port argument",\
notification = "direct", \
log_message= "start GLB: lost iplist_port argument",\
response = "please input iplist and server port<ip:port>.")
if port is None or port == '':
raise HTTPAPIError(status_code=417, error_detail="lost port argument",\
notification = "direct", \
log_message= "start GLB: lost port argument",\
response = "please input port.")
if args is None:
args = ''
#mysql:{"User": "******","Pass": "******","Addr": "0.0.0.0","Port": "3306","Backend": ["10.200.86.14:3306","10.200.86.15:3306","10.200.86.16:3306"]}
#manager:{"Addr": "0.0.0.0","Port": "8888","Service":"http","Backend": ["10.200.86.74:8888","10.200.86.75:8888","10.200.86.76:8888"]}
glb_config = {}
if user:
glb_config['User'] = user
if passwd:
glb_config['Pass'] = passwd
if service and service != 'mysql':
glb_config['Service'] = service
glb_config['Addr'] = '0.0.0.0'
glb_config['Port'] = port
ips = iplist_port.split(",")
glb_config['Backend'] = ips
if service == 'mysql':
for ip in ips:
if not ":" in ip:
raise HTTPAPIError(status_code=417, error_detail="iplist_port input error",\
notification = "direct", \
log_message= "iplist_port input error", \
response = "please check iplist_port")
i, p = ip.split(':')
try:
conn = MySQLdb.Connect(host=i,
user=user,
passwd=passwd,
port=int(p))
except Exception, e:
conn = None
if conn is None:
raise HTTPAPIError(status_code=417, error_detail="cann't connect to mysql",\
notification = "direct", \
log_message= "cann't connect to mysql", \
response = "please check mysqld or user password")
if not os.path.exists(os.path.dirname(options.glb_json_file_name)):
os.mkdir(os.path.dirname(options.glb_json_file_name))
config_file = options.glb_json_file_name % (port)
cmd = r'ps -ef|grep %s|grep -v grep' % (config_file)
result = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
if not result[0] == '':
raise HTTPAPIError(status_code=417, error_detail="this glb is running",\
notification = "direct", \
log_message= "this glb is running", \
response = "please check envirment")
f = open(config_file, 'w')
logging.info("start gbalancer: %s" % (glb_config))
f.write(json.dumps(glb_config, sort_keys=True, indent=4))
f.flush()
f.close()
#self.invokeCommand.run_service_shell(options.start_gbalancer % (config_file, args))
self.invokeCommand.run_check_shell(options.start_gbalancer %
(config_file, args))
time.sleep(1)
glb_proc = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
logging.info("glb_proc: %s" % str(glb_proc))
if len(glb_proc) != 1 or glb_proc[0] == '':
#if len(glb_proc) == 0:
raise HTTPAPIError(status_code=417, error_detail="glb start error",\
notification = "direct", \
log_message= "glb start error", \
response = "please check envirment")
dict = {}
#dict.setdefault("code", '000000')
dict.setdefault("message", "start gbalancer successful!")
self.finish(dict)
class DBStatOpers(Abstract_Stat_Service):
dba_opers = DBAOpers()
confOpers = ConfigFileOpers()
def __init__(self):
'''
Constructor
'''
'''
@todo: why use str_flag?
'''
def stat(self, str_flag=""):
# if str_flag = "", then the request must come from the webport, not peer.
if str_flag == "":
rows_oper_dict = self._stat_rows_oper()
# We check if the local database is in use.
if (rows_oper_dict == {}
or rows_oper_dict["oper_total"]["num_reads"] == 0):
result_dict = self.get_peer_wsrep_status()
logging.info("rows_oper_dict:" + str(rows_oper_dict))
# When local database is in use, we go on processing in this node.
else:
wsrep_status_dict = self._stat_wsrep_status()
innodb_buffer_dict = self._stat_innodb_buffer()
variable_status_dict = self._stat_variable_status()
else:
# This str_flag must be "inner", This process the request from peer nodes.
wsrep_status_dict = self._stat_wsrep_status()
rows_oper_dict = self._stat_rows_oper()
innodb_buffer_dict = self._stat_innodb_buffer()
variable_status_dict = self._stat_variable_status()
# If we find that the local database is not in use, then the all results come from peer node.
if (rows_oper_dict == {}
or rows_oper_dict["oper_total"]["num_reads"] == 0):
return result_dict
result = {}
# Else we know that local database in in use, we return it in the original way.
result.setdefault('wsrep_status_dict', wsrep_status_dict)
result.setdefault('rows_oper', rows_oper_dict)
result.setdefault('innodb_buffer', innodb_buffer_dict)
result.setdefault('variable_status', variable_status_dict)
return result
def get_peer_wsrep_status(self):
logging.info("can not connect to local database site")
cluster_started_nodes = self.zkOper.retrieve_started_nodes()
confDict = self.confOpers.getValue(options.data_node_property,
['dataNodeIp'])
local_ip = confDict['dataNodeIp']
logging.info("local ip:" + str(local_ip))
if cluster_started_nodes.count(local_ip) != 0:
cluster_started_nodes.remove(local_ip)
logging.info("candicates are: " + str(cluster_started_nodes))
result = ""
for ip in cluster_started_nodes:
url_post = "/db/all/stat?inner=true"
result = self.communicate(ip, url_post)
logging.info("origin result: " + str(result))
logging.info(result.replace("\\", ""))
if result.count("wsrep_status_dict") != 0:
break
if result.count("wsrep_status_dict") != 0:
result_dict = json.loads(result)
return result_dict["response"]
else:
raise CommonException("Can\'t connect to mysql server")
def _stat_wsrep_status(self):
conn = self.dba_opers.get_mysql_connection()
if conn is None:
raise CommonException("Can\'t connect to mysql server")
try:
rows = self.dba_opers.show_status(conn)
finally:
conn.close()
key_value = retrieve_kv_from_db_rows(rows,['wsrep_flow_control_paused',\
'wsrep_flow_control_sent',\
'wsrep_local_recv_queue_avg',\
'wsrep_local_send_queue_avg'])
slowest_node_param_dict = {}
slowest_node_param_dict.setdefault(
'wsrep_flow_control_sent',
key_value.get('wsrep_flow_control_sent'))
slowest_node_param_dict.setdefault(
'wsrep_local_recv_queue_avg',
key_value.get('wsrep_local_recv_queue_avg'))
result = {}
result.setdefault('wsrep_flow_control_paused',
key_value.get('wsrep_flow_control_paused'))
result.setdefault('slowest_node_param', slowest_node_param_dict)
result.setdefault('wsrep_local_send_queue_avg',
key_value.get('wsrep_local_send_queue_avg'))
return result
def communicate(self, peer_ip, url):
http_client = tornado.httpclient.HTTPClient()
requesturi = "http://" + peer_ip + ":" + str(options.port) + url
try:
response = http_client.fetch(requesturi)
except tornado.httpclient.HTTPError as e:
logging.error(str(e))
http_client.close()
return "error"
logging.info(str(response.body))
return response.body
def stat_wsrep_status_flow_control_paused(self):
status_dict = self._stat_wsrep_status()
value = status_dict.get('wsrep_flow_control_paused')
return {'wsrep_flow_control_paused': value}
def stat_wsrep_status_slowest_node_param(self):
status_dict = self._stat_wsrep_status()
sub_dict = status_dict.get('slowest_node_param')
return sub_dict
def stat_wsrep_status_slowest_network_param(self):
status_dict = self._stat_wsrep_status()
value = status_dict.get('wsrep_local_send_queue_avg')
return {'wsrep_local_send_queue_avg': value}
@run_on_executor()
@run_callback
def stat_binlog_eng_log_pos(self, params):
if not params:
raise UserVisiableException('params are not given')
conn = self.dba_opers.get_mysql_connection()
if None == conn:
raise UserVisiableException("Can\'t connect to mysql server")
log_pos_info = ''
master_log_file = ''
end_log_pos = ''
try:
cursor = conn.cursor()
cursor.execute('show binary logs')
rows_bin_logs = cursor.fetchall()
assert rows_bin_logs
invokecommand = InvokeCommand()
for i in range(len(rows_bin_logs)):
master_log_file = rows_bin_logs[-i - 1][-2]
ret_str = invokecommand._runSysCmd(
'''mysql -uroot -pMcluster -e "show binlog events IN '%s'"|grep 'xid=%s' '''
% (master_log_file, params['xid']))
assert ret_str
log_pos_info = ret_str[0]
if log_pos_info:
break
end_log_pos = log_pos_info.strip('\n').split('\t')[-2]
finally:
conn.close()
result = {}
result.setdefault('Master_Log_File', master_log_file)
result.setdefault('End_Log_Pos', end_log_pos)
return result
@run_on_executor()
@run_callback
def bin_log_node_stat(self):
conn = self.dba_opers.get_mysql_connection()
if None == conn:
raise UserVisiableException("Can\'t connect to mysql server")
try:
cursor = conn.cursor()
cursor.execute("show variables like 'log_bin'")
rows_stat_log_bin = cursor.fetchall()
stat_log_bin = rows_stat_log_bin[0][1]
finally:
conn.close()
zkOper = self.retrieve_zkOper()
started_node_list = zkOper.retrieve_started_nodes()
local_ip = get_localhost_ip()
if local_ip in started_node_list:
started_node_list.remove(local_ip)
result = {}
result.setdefault('node_list', started_node_list)
result.setdefault('stat_log_bin', stat_log_bin)
return result
def _stat_rows_oper(self):
processor_existed = self._check_mysql_processor_exist()
result = {}
if not processor_existed:
return result
target_dict = self._retrieve_dict_with_result(options.stat_rows_oper)
key_list = ['num_updates', 'num_reads', 'num_deletes', 'num_inserts']
oper_total_dict = self._split_key_value(key_list, target_dict)
key_list = [
'num_reads_sec', 'num_updates_sec', 'num_deletes_sec',
'num_inserts_sec'
]
oper_per_second_dict = self._split_key_value(key_list, target_dict)
result.setdefault("oper_total", oper_total_dict)
result.setdefault("oper_per_second", oper_per_second_dict)
return result
def stat_rows_oper_total(self):
oper_dict = self._stat_rows_oper()
sub_dict = oper_dict.get('oper_total')
return sub_dict
def stat_rows_oper_per_second(self):
oper_dict = self._stat_rows_oper()
sub_dict = oper_dict.get('oper_per_second')
return sub_dict
def _stat_innodb_buffer(self):
processor_existed = self._check_mysql_processor_exist()
result = {}
if not processor_existed:
return result
target_dict = self._retrieve_dict_with_result(
options.stat_innodb_buffer)
key_list = ['total_mem_alloc', 'add_pool_alloc']
mem_alloc_dict = self._split_key_value(key_list, target_dict)
key_list = ['pages_total', 'pages_modified']
page_dict = self._split_key_value(key_list, target_dict)
key_list = ['buf_pool_size', 'buf_pool_hit_rate', 'buf_free']
buffer_pool_dict = self._split_key_value(key_list, target_dict)
value = buffer_pool_dict.get('buf_pool_hit_rate')
if value == '--' or value == '' or value == 0:
value = 0
else:
buf_pool_hit_rate_list = value.split('/')
value = int(buf_pool_hit_rate_list[0]) / int(
buf_pool_hit_rate_list[1])
buffer_pool_dict['buf_pool_hit_rate'] = str(value)
result.setdefault("mem_alloc", mem_alloc_dict)
result.setdefault("page", page_dict)
result.setdefault("buffer_pool", buffer_pool_dict)
return result
def stat_innodb_buffer_mem_alloc(self):
buffer_dict = self._stat_innodb_buffer()
sub_dict = buffer_dict.get('mem_alloc')
return sub_dict
def stat_innodb_buffer_page(self):
buffer_dict = self._stat_innodb_buffer()
sub_dict = buffer_dict.get('page')
return sub_dict
def stat_innodb_buffer_buffer_pool(self):
buffer_dict = self._stat_innodb_buffer()
sub_dict = buffer_dict.get('buffer_pool')
return sub_dict
def _stat_variable_status(self):
processor_existed = self._check_mysql_processor_exist()
result = {}
if not processor_existed:
return result
target_dict = self._retrieve_dict_with_result(
options.stat_variable_status)
key_list = ['Opens_PS', 'QPS', 'Commit_PS', 'Threads_PS']
ps_dict = self._split_key_value(key_list, target_dict)
key_list = [
'Thread_Cache_Used', 'CXN_Used_Ever', 'CXN_Used_Now',
'Table_Cache_Used'
]
used_dict = self._split_key_value(key_list, target_dict)
key_list = ['R_W_Ratio', 'Rollback_Commit', 'Write_Commit']
ratio_dict = self._split_key_value(key_list, target_dict)
result.setdefault("ps", ps_dict)
result.setdefault("used", used_dict)
result.setdefault("ration", ratio_dict)
return result
def stat_variable_status_ps(self):
status_dict = self._stat_variable_status()
sub_dict = status_dict.get('ps')
return sub_dict
def stat_variable_status_used(self):
status_dict = self._stat_variable_status()
sub_dict = status_dict.get('used')
return sub_dict
def stat_variable_status_ration(self):
status_dict = self._stat_variable_status()
sub_dict = status_dict.get('ration')
return sub_dict
class DataNodeToMCluster(APIHandler):
confOpers = ConfigFileOpers()
def post(self):
requestParam = {}
args = self.request.arguments
logging.info("args :" + str(args))
if not args:
raise HTTPAPIErrorException("params is empty")
for key in args:
value = args[key][0]
requestParam.setdefault(key, value)
if "dataNodeName" not in requestParam or "dataNodeIp" not in requestParam:
raise HTTPAPIErrorException("dataNodeName or dataNodeIp is empty, please check it!")
if self.confOpers.ipFormatChk(requestParam['dataNodeIp']):
raise HTTPAPIErrorException("dataNodeIp is illegal", status_code=417)
self.confOpers.setValue(options.data_node_property, requestParam)
dataNodeProprs = self.confOpers.getValue(options.data_node_property)
zkOper = self.retrieve_watch_zkOper()
clusterUUID = zkOper.getClusterUUID()
zkOper.writeDataNodeInfo(clusterUUID, dataNodeProprs)
data, _ = zkOper.retrieveClusterProp(clusterUUID)
self.confOpers.setValue(options.cluster_property, eval(data))
fullText, _ = zkOper.retrieveMysqlProp(clusterUUID)
self.confOpers.writeFullText(options.mysql_cnf_file_name, fullText)
data_node_ip = requestParam.get('dataNodeIp')
mycnfParam = self.confOpers.getValue(options.mysql_cnf_file_name)
orginal_cluster_address = mycnfParam['wsrep_cluster_address']
index = orginal_cluster_address.find("//")
ip_str = orginal_cluster_address[index + 2:]
ip_lists = ip_str.rstrip().split(",")
if data_node_ip in ip_lists:
error_message = "this node have add to cluster, no need to add it!"
raise HTTPAPIErrorException(error_message, status_code=417)
new_cluster_address = orginal_cluster_address + "," + str(data_node_ip)
data_node_name = requestParam.get('dataNodeName')
# mysql_cnf_full_text = self.confOpers.retrieveFullText(options.mysql_cnf_file_name)
# self.confOpers.writeFullText(options.mysql_cnf_file_name, mysql_cnf_full_text)
keyValueMap = {}
keyValueMap.setdefault('wsrep_cluster_address', new_cluster_address)
keyValueMap.setdefault('wsrep_node_name', str(data_node_name))
keyValueMap.setdefault('wsrep_node_address', str(data_node_ip))
self.confOpers.setValue(options.mysql_cnf_file_name, keyValueMap)
mysql_cnf_full_text = self.confOpers.retrieveFullText(options.mysql_cnf_file_name)
zkOper.writeMysqlCnf(clusterUUID, mysql_cnf_full_text)
result = {}
# dict.setdefault("code", "000000")
result.setdefault("message", "add data node into cluster successful!")
self.finish(result)
def delete(self):
mysql_service_opers = Node_Mysql_Service_Opers()
mysql_service_opers.stop()
dataNodeProprs = self.confOpers.getValue(options.data_node_property)
ip = dataNodeProprs['dataNodeIp']
mycnfParam = self.confOpers.getValue(options.mysql_cnf_file_name)
orginal_cluster_address = mycnfParam['wsrep_cluster_address']
index = orginal_cluster_address.find("//")
prefix = orginal_cluster_address[:index + 2]
ipStr = orginal_cluster_address[index + 2:]
ipLists = ipStr.rstrip().split(",")
assert ip in ipLists
ipLists.remove(ip)
removeRes = prefix + ','.join(ipLists)
self.confOpers.setValue(options.mysql_cnf_file_name, {
'wsrep_cluster_address': removeRes})
newMyConfText = self.confOpers.retrieveFullText(
options.mysql_cnf_file_name)
zkOper = self.retrieve_watch_zkOper()
clusterUUID = zkOper.getClusterUUID()
zkOper.writeMysqlCnf(clusterUUID, newMyConfText)
zkOper.remove_data_node_name(ip)
zkOper.remove_started_node(ip)
self.finish({"message": "remove data node from cluster successful!"})
from tornado.options import options
from kazoo.client import KazooClient, KazooState
from kazoo.exceptions import SessionExpiredError, NoNodeError
from kazoo.handlers.threading import TimeoutError
from kazoo.retry import KazooRetry
from common.utils.exceptions import CommonException
from common.my_logging import debug_log
from common.utils.decorators import singleton, timeout_handler
from common.utils import local_get_zk_address, CLUSTER_NAME
from common.configFileOpers import ConfigFileOpers
from common.helper import getDictFromText
log_obj = debug_log('zkOpers')
logger = log_obj.get_logger_object()
confOpers = ConfigFileOpers()
class ZkOpers(object):
zk = None
DEFAULT_RETRY_POLICY = KazooRetry(
max_tries=None,
max_delay=10000,
)
rootPath = "/letv/mysql/mcluster"
'''
classdocs
'''
class DBUser(APIHandler):
dba_opers = DBAOpers()
conf_opers = ConfigFileOpers()
def post(self):
role = self.get_argument("role", None)
dbName = self.get_argument("dbName", None)
userName = self.get_argument("userName", None)
user_password = self.get_argument("user_password", None)
ip_address = self.get_argument("ip_address", '%')
max_queries_per_hour = self.get_argument("max_queries_per_hour", 0)
max_updates_per_hour = self.get_argument("max_updates_per_hour", 0)
max_connections_per_hour = self.get_argument("max_connections_per_hour", 0)
max_user_connections = self.get_argument("max_user_connections", 200)
dict = {}
dict = self.request.arguments
if dict.has_key("user_password"):
del dict["user_password"]
logging.info(str(dict))
if role is None:
raise HTTPAPIErrorException("when create db's user, no specify the user role, please specify the user role.",
status_code=417)
if dbName is None:
raise HTTPAPIErrorException("when create db's user, no specify the database name, please specify the database name.",
status_code=417)
if userName is None:
raise HTTPAPIErrorException("when create db's user, no specify the user name, please specify the user name.",
status_code=417)
if ip_address is None:
raise HTTPAPIErrorException("when create db's user, no specify the ip address, please specify the ip address.",
status_code=417)
if user_password is None:
user_password = get_random_password()
existed_flag = "true"
conn = self.dba_opers.get_mysql_connection()
try:
existed_flag = self.dba_opers.check_if_existed_database(conn, dbName)
if existed_flag == "false":
raise HTTPAPIErrorException("Please create database " + dbName + " first",
status_code=417)
self.dba_opers.create_user(conn, userName, user_password, ip_address)
if 'manager' == role:
self.dba_opers.grant_manager_privileges(conn, userName, user_password, dbName, ip_address,
max_queries_per_hour,
max_updates_per_hour,
max_connections_per_hour,
max_user_connections)
elif 'wr' == role:
self.dba_opers.grant_wr_privileges(conn, userName, user_password, dbName, ip_address,
max_queries_per_hour,
max_updates_per_hour,
max_connections_per_hour,
max_user_connections)
elif 'ro' == role:
max_updates_per_hour = 1
self.dba_opers.grant_readonly_privileges(conn, userName, user_password, dbName, ip_address,
max_queries_per_hour,
max_connections_per_hour,
max_user_connections)
else:
# use try catch to close the conn
# conn.close()
raise HTTPAPIErrorException("please valid the specified role, the type is [manager,ro,wr]", status_code=417)
self.dba_opers.flush_privileges(conn)
finally:
conn.close()
# check if exist cluster
zkOper = self.retrieve_zkOper()
clusterUUID = zkOper.getClusterUUID()
userProps = {'role': role,
'max_queries_per_hour': max_queries_per_hour,
'max_updates_per_hour': max_updates_per_hour,
'max_connections_per_hour': max_connections_per_hour,
'max_user_connections': max_user_connections}
zkOper.write_user_info(clusterUUID, dbName, userName, ip_address, userProps)
result = {}
# dict.setdefault("code", '000000')
result.setdefault("message", "user has been created successful!")
result.setdefault("user_role", role)
result.setdefault("user_name", userName)
result.setdefault("user_password", user_password)
self.finish(result)
def put(self):
dbName = self.get_argument("dbName", None)
userName = self.get_argument("userName", None)
ip_address = self.get_argument("ip_address", '%')
max_queries_per_hour = self.get_argument("max_queries_per_hour", None)
max_updates_per_hour = self.get_argument("max_updates_per_hour", None)
max_connections_per_hour = self.get_argument("max_connections_per_hour", None)
max_user_connections = self.get_argument("max_user_connections", None)
role = self.get_argument("role", None)
if dbName is None:
raise HTTPAPIErrorException("when modify db's user, no specify the database name, please specify the database name.",
status_code=417)
if userName is None:
raise HTTPAPIErrorException("when modify db's user, no specify the user name, please specify the user name.",
status_code=417)
if ip_address is None:
raise HTTPAPIErrorException("when modify db's user, no specify the ip address, please specify the ip address.",
status_code=417)
if max_queries_per_hour is None and max_updates_per_hour is None and max_connections_per_hour is None and max_user_connections is None:
raise HTTPAPIErrorException("when modify db's user, no specify any modified parameter, please specify the ip address.\
please specify any one or all of following parameter:[max_queries_per_hour,\
max_updates_per_hour,max_connections_per_hour,max_user_connections]",
status_code=417)
if role is None:
raise HTTPAPIErrorException("when modify db's user, no specify the role, please specify the role.",
status_code=417)
conn = self.dba_opers.get_mysql_connection()
try:
zkOper = self.retrieve_zkOper()
clusterUUID = zkOper.getClusterUUID()
user_limit_map = {}
if not max_queries_per_hour or not max_updates_per_hour or not max_connections_per_hour or not max_user_connections:
user_limit_map = zkOper.retrieve_user_limit_props(clusterUUID, dbName, userName, ip_address)
if not user_limit_map:
raise HTTPAPIErrorException("when modify db's user, no found specified user!\
please check the valid of the specified user, because the system no found the user!",
status_code=417)
if max_queries_per_hour is None:
max_queries_per_hour = user_limit_map.get('max_queries_per_hour')
if max_updates_per_hour is None:
max_updates_per_hour = user_limit_map.get('max_updates_per_hour')
if max_connections_per_hour is None:
max_connections_per_hour = user_limit_map.get('max_connections_per_hour')
if max_user_connections is None:
max_user_connections = user_limit_map.get('max_user_connections')
self.dba_opers.grant_resource_limit(conn, userName, dbName, ip_address, role,
max_queries_per_hour,
max_updates_per_hour,
max_connections_per_hour,
max_user_connections)
self.dba_opers.flush_privileges(conn)
userProps = {'role': user_limit_map.get('role'),
'max_queries_per_hour': max_queries_per_hour,
'max_updates_per_hour': max_updates_per_hour,
'max_connections_per_hour': max_connections_per_hour,
'max_user_connections': max_user_connections}
zkOper.write_user_info(clusterUUID, dbName, userName, ip_address, userProps)
finally:
conn.close()
result = {}
result.setdefault("message", "modify the user's resource limit successfully!")
result.setdefault("db_name", dbName)
result.setdefault("user_name", userName)
self.finish(result)
def delete(self, dbName, userName, ipAddress):
if not dbName:
raise HTTPAPIErrorException("when remove db's user, no specify the database name,\
please specify the database name.",
status_code=417)
if not userName:
raise HTTPAPIErrorException("when remove db's user, no specify the user name,\
please specify the user name.",
status_code=417)
if not ipAddress:
raise HTTPAPIErrorException("when remove db's user, no specify the ip address,\
please specify the ip address.",
status_code=417)
conn = self.dba_opers.get_mysql_connection()
try:
self.dba_opers.delete_user(conn, userName, ipAddress)
finally:
conn.close()
# check if exist cluster
zkOper = self.retrieve_zkOper()
clusterUUID = zkOper.getClusterUUID()
zkOper.remove_db_user(clusterUUID, dbName, userName, ipAddress)
result = {}
result.setdefault("message", "removed user successfully!")
result.setdefault("user_name", userName)
result.setdefault("ip_address", ipAddress)
self.finish(result)
