def run(self):
exc_info = None
try:
while not self.threading_exception_queue.empty():
exc_info = self.threading_exception_queue.get(block=False)
if exc_info is None:
continue
e = exc_info[1]
if isinstance(e, HTTPAPIError):
pass
elif isinstance(e, HTTPError):
e = HTTPAPIError(e.status_code)
else:
e = HTTPAPIError(500)
exception = "".join([ln for ln in traceback.format_exception(*exc_info)])
logging.error(e)
self._send_error_email(exception)
except:
exc_type, exc_obj, exc_trace = exc_info
# deal with the exception
logging.error(exc_type)
logging.error(exc_obj)
logging.error(exc_trace)
def write_error(self, status_code, **kwargs):
"""Override to implement custom error pages."""
debug = self.settings.get("debug", False)
try:
exc_info = kwargs.pop('exc_info')
e = exc_info[1]
if isinstance(e, HTTPAPIError):
pass
elif isinstance(e, HTTPError):
e = HTTPAPIError(e.status_code)
else:
e = HTTPAPIError(500)
exception = "".join(
[ln for ln in traceback.format_exception(*exc_info)])
if status_code == 500 and not debug:
self.logger.error(e)
self._send_error_email(exception)
if debug:
e.response["exception"] = exception
self.clear()
self.set_status(200) # always return 200 OK for API errors
self.set_header("Content-Type", "application/json; charset=UTF-8")
self.finish(str(e))
except Exception:
self.logger.error(traceback.format_exc())
return super(APIHandler, self).write_error(status_code, **kwargs)
def post(self):
port = self.get_argument('port', None)
if port is None or port == '':
raise HTTPAPIError(status_code=417, error_detail="lost port argument",\
notification = "direct", \
log_message= "start GLB: lost port argument",\
response = "please input port.")
cmd = r'netstat -ntlp|grep -w %s|grep -v grep' % (port)
result = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
if result[0] == '':
raise HTTPAPIError(status_code=417, error_detail="this glb is stopped",\
notification = "direct", \
log_message= "this glb is stopped", \
response = "please check envirment")
logging.info("stop gbalancer(port): %s" % (port))
temp = re.split('\s+', result[0])
glb_pid = temp[len(temp) - 1].split('/')[0]
self.invokeCommand.run_check_shell(r'kill -9 %s' % (glb_pid))
glb_proc = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
if not glb_proc[0] == '':
raise HTTPAPIError(status_code=417, error_detail="cannot stop glb",\
notification = "direct", \
log_message= "cannot stop glb", \
response = "please check envirment")
dict = {}
#dict.setdefault("code", '000000')
dict.setdefault("message", "stop gbalancer successful!")
self.finish(dict)
def run(self):
invokeCommand = InvokeCommand()
now = datetime.datetime.now()
logging.info("Backup start time is " + now.strftime(TIME_FORMAT))
if self._backup_mode == "full":
ret_sub_p = invokeCommand._runSysCmdnoWait(options.full_back_sh)
ret_str = ret_sub_p.stdout.read()
logging.info("ret_sub_p" + str(ret_sub_p))
if ret_sub_p == False:
raise HTTPAPIError(status_code=411, error_detail="Full Back up process terminated!",\
notification = "direct", \
log_message= "Full Back up process terminated!",\
response = "Full Back up process terminated!")
else:
ret_sub_p = invokeCommand._runSysCmdnoWait(options.inc_back_sh)
if ret_sub_p == False:
raise HTTPAPIError(status_code=411, error_detail="Increment Back up process terminated!",\
notification = "direct", \
log_message= "Increment Back up process terminated!",\
response = "Increment Back up process terminated!")
logging.info("wait here")
ret_sub_p.wait()
def post(self):
"""params include: url, 'cron' or 'interval' or 'date', priority
"""
args = self.get_all_arguments()
print 'args: %s' % str(args)
try:
params = self.__verify_params(args)
except Exception as err_msg:
raise HTTPAPIError(status_code=400, error_detail=err_msg,\
notification = "direct", \
log_message= err_msg,\
response = "please check params!")
logging.info('get params:%s' % str(params))
self.http_request_opers.add_http_job(params)
result = {}
result.setdefault(
'message',
'add job in queue :%s successfully!' % options.queue_http_request)
self.finish(result)
def prepare(self):
super(APIErrorHandler, self).prepare()
raise HTTPAPIError(404)
def _issue_start_action(self, cluster_flag, cluster_mode):
node_wsrep_status_dict = {}
data_node_started_flag_dict = {}
need_start_node_ip_list = []
arbitrator_node_ip = []
status_dict = {}
try:
data_node_info_list = self.zkOper.retrieve_data_node_list()
node_num = len(data_node_info_list)
adminUser, adminPasswd = _retrieve_userName_passwd()
if None != cluster_mode:
mode_dict = {"cluster_mode": cluster_mode}
self.zkOper.writeClusterMode(mode_dict)
if cluster_flag == 'new':
status_dict.setdefault("_status", "initializing")
self.zkOper.writeClusterStatus(status_dict)
portstatus_obj = PortStatus()
need_start_node_ip_list = portstatus_obj.check_port(
data_node_info_list)
logging.info("need_start_node_ip_list:" +
str(need_start_node_ip_list))
if node_num - len(need_start_node_ip_list) != 1:
error_message = "data nodes's status is abnormal."
status_dict['_status'] = 'failed'
self.zkOper.writeClusterStatus(status_dict)
# logging.error("Some nodes's status are abnormal")
raise CommonException(error_message)
'''
@todo: for arbitrator mode? need this code?
'''
if cluster_mode == "asymmetric":
arbitrator_node = Arbitrator()
arbitrator_ip = arbitrator_node.get_ip(data_node_info_list)
arbitrator_node_ip.append(arbitrator_ip)
need_start_node_ip_list.remove(arbitrator_ip)
else:
wsrepstatus_obj = WsrepStatus()
w_num = wsrepstatus_obj.check_wsrep(data_node_info_list,
node_wsrep_status_dict,
node_num)
if w_num == node_num:
error_message = "all data node's status is ok. No need to start them."
raise CommonException(error_message)
logging.info(
"Check the data node wsrep status for start cluster, the wsrep status value is %s"
% str(node_wsrep_status_dict))
data_node_stop_finished_flag_dict = {}
stop_issue_obj = StopIssue()
stop_finished_count = stop_issue_obj.issue_stop(
node_wsrep_status_dict, data_node_stop_finished_flag_dict,
adminUser, adminPasswd)
if stop_finished_count == len(
data_node_stop_finished_flag_dict):
status_dict['_status'] = 'stopped'
self.zkOper.writeClusterStatus(status_dict)
self._send_email(
"mcluster",
" mysql service have been stopped in the cluster")
logging.info('nodes stopping finished!')
_galerastatus_obj = GaleraStatus()
uuid_seqno_dict = _galerastatus_obj.check_status(
data_node_stop_finished_flag_dict)
err_node = ""
for (node_ip, value) in uuid_seqno_dict.items():
if not value:
err_node.join(node_ip)
if err_node != "":
error_message = "data node(%s) error, please check the status and start it by human." % (
node_ip)
status_dict['_status'] = 'failed'
self.zkOper.writeClusterStatus(status_dict)
raise CommonException(error_message)
need_start_node_ip_list = self._sort_seqno(uuid_seqno_dict)
logging.info("After sort, the uuid_seqno_dict value is %s" %
str(need_start_node_ip_list))
url_post = "/node/start"
logging.info("/node/start start issue!")
logging.info("need_start_node_ip_list:" +
str(need_start_node_ip_list))
for data_node_ip in need_start_node_ip_list:
started_nodes = self.zkOper.retrieve_started_nodes()
# started_nodes_count = len(started_nodes)
isNewCluster = False
if len(started_nodes) == 0:
isNewCluster = True
args_dict = {}
args_dict.setdefault("isNewCluster", str(isNewCluster))
requesturi = "http://%s:%s%s" % (data_node_ip, options.port,
url_post)
request = HTTPRequest(url=requesturi,
method='POST',
body=urllib.urlencode(args_dict),
auth_username=adminUser,
auth_password=adminPasswd)
_request_fetch(request, timeout=100)
start_finished = self._check_start_status(
data_node_ip, cluster_flag)
logging.info('request from node/start is ' +
str(start_finished))
if start_finished == False:
status_dict['_status'] = 'failed'
self.zkOper.writeClusterStatus(status_dict)
error_message = "%s'database status is failed!" % (
data_node_ip)
raise CommonException(error_message)
logging.info("check started nodes ok!")
data_node_started_flag_dict.setdefault(data_node_ip,
start_finished)
'''
@todo: for arbitrator need this code?
'''
if cluster_mode == "asymmetric":
arbitrator_ip = arbitrator_node_ip[0]
url_post = "arbitrator/node/start"
requesturi = "http://%s:%s%s" % (arbitrator_ip,
options.port, url_post)
request = HTTPRequest(url=requesturi,
method='POST',
body=urllib.urlencode(args_dict),
auth_username=adminUser,
auth_password=adminPasswd)
logging.info("issue " + requesturi)
return_result = _request_fetch(request)
if return_result == False:
raise HTTPAPIError("Garbd arbitrator start failed", \
notification = "direct", \
log_message="garbd arbitrator start failed",
response = "garbd arbitrator start failed")
else:
self.zkOper.write_started_node(arbitrator_ip)
cluster_started_nodes_list = self.zkOper.retrieve_started_nodes()
nodes_online = len(cluster_started_nodes_list)
logging.info("starts nodes" + str(nodes_online))
if nodes_online == node_num:
status_dict['_status'] = 'running'
self._send_email(
"mcluster",
" mysql services have been started in the cluster")
else:
status_dict['_status'] = 'failed'
self._send_email("mcluster",
" mysql services have benn started error")
self.zkOper.writeClusterStatus(status_dict)
except Exception, e:
logging.error(e)
status_dict['_status'] = 'failed'
self.zkOper.writeClusterStatus(status_dict)
raise e
def post(self):
user = self.get_argument('user', None)
passwd = self.get_argument('passwd', None)
service = self.get_argument('service', None)
iplist_port = self.get_argument('iplist_port', None)
port = self.get_argument('port', None)
args = self.get_argument('args', None)
if service is None:
raise HTTPAPIError(status_code=417, error_detail="lost service arguments", \
notification = "direct", \
log_message= "start GLB: lost service argument",\
response = "please input service.")
if service == 'mysql':
lost_args = []
if user is None:
lost_args.append("user")
if passwd is None:
lost_args.append("passwd")
if len(lost_args) != 0:
raise HTTPAPIError(status_code=417, error_detail="lost %s arguments" % (lost_args), \
notification = "direct", \
log_message= "start GLB: lost %s argument" % (lost_args),\
response = "please input %s." % (lost_args))
if iplist_port is None or iplist_port == '':
raise HTTPAPIError(status_code=417, error_detail="lost iplist_port argument",\
notification = "direct", \
log_message= "start GLB: lost iplist_port argument",\
response = "please input iplist and server port<ip:port>.")
if port is None or port == '':
raise HTTPAPIError(status_code=417, error_detail="lost port argument",\
notification = "direct", \
log_message= "start GLB: lost port argument",\
response = "please input port.")
if args is None:
args = ''
#mysql:{"User": "******","Pass": "******","Addr": "0.0.0.0","Port": "3306","Backend": ["10.200.86.14:3306","10.200.86.15:3306","10.200.86.16:3306"]}
#manager:{"Addr": "0.0.0.0","Port": "8888","Service":"http","Backend": ["10.200.86.74:8888","10.200.86.75:8888","10.200.86.76:8888"]}
glb_config = {}
if user:
glb_config['User'] = user
if passwd:
glb_config['Pass'] = passwd
if service and service != 'mysql':
glb_config['Service'] = service
glb_config['Addr'] = '0.0.0.0'
glb_config['Port'] = port
ips = iplist_port.split(",")
glb_config['Backend'] = ips
if service == 'mysql':
for ip in ips:
if not ":" in ip:
raise HTTPAPIError(status_code=417, error_detail="iplist_port input error",\
notification = "direct", \
log_message= "iplist_port input error", \
response = "please check iplist_port")
i, p = ip.split(':')
try:
conn = MySQLdb.Connect(host=i,
user=user,
passwd=passwd,
port=int(p))
except Exception, e:
conn = None
if conn is None:
raise HTTPAPIError(status_code=417, error_detail="cann't connect to mysql",\
notification = "direct", \
log_message= "cann't connect to mysql", \
response = "please check mysqld or user password")
class Startgbalancer(APIHandler):
confOpers = ConfigFileOpers()
invokeCommand = InvokeCommand()
def post(self):
user = self.get_argument('user', None)
passwd = self.get_argument('passwd', None)
service = self.get_argument('service', None)
iplist_port = self.get_argument('iplist_port', None)
port = self.get_argument('port', None)
args = self.get_argument('args', None)
if service is None:
raise HTTPAPIError(status_code=417, error_detail="lost service arguments", \
notification = "direct", \
log_message= "start GLB: lost service argument",\
response = "please input service.")
if service == 'mysql':
lost_args = []
if user is None:
lost_args.append("user")
if passwd is None:
lost_args.append("passwd")
if len(lost_args) != 0:
raise HTTPAPIError(status_code=417, error_detail="lost %s arguments" % (lost_args), \
notification = "direct", \
log_message= "start GLB: lost %s argument" % (lost_args),\
response = "please input %s." % (lost_args))
if iplist_port is None or iplist_port == '':
raise HTTPAPIError(status_code=417, error_detail="lost iplist_port argument",\
notification = "direct", \
log_message= "start GLB: lost iplist_port argument",\
response = "please input iplist and server port<ip:port>.")
if port is None or port == '':
raise HTTPAPIError(status_code=417, error_detail="lost port argument",\
notification = "direct", \
log_message= "start GLB: lost port argument",\
response = "please input port.")
if args is None:
args = ''
#mysql:{"User": "******","Pass": "******","Addr": "0.0.0.0","Port": "3306","Backend": ["10.200.86.14:3306","10.200.86.15:3306","10.200.86.16:3306"]}
#manager:{"Addr": "0.0.0.0","Port": "8888","Service":"http","Backend": ["10.200.86.74:8888","10.200.86.75:8888","10.200.86.76:8888"]}
glb_config = {}
if user:
glb_config['User'] = user
if passwd:
glb_config['Pass'] = passwd
if service and service != 'mysql':
glb_config['Service'] = service
glb_config['Addr'] = '0.0.0.0'
glb_config['Port'] = port
ips = iplist_port.split(",")
glb_config['Backend'] = ips
if service == 'mysql':
for ip in ips:
if not ":" in ip:
raise HTTPAPIError(status_code=417, error_detail="iplist_port input error",\
notification = "direct", \
log_message= "iplist_port input error", \
response = "please check iplist_port")
i, p = ip.split(':')
try:
conn = MySQLdb.Connect(host=i,
user=user,
passwd=passwd,
port=int(p))
except Exception, e:
conn = None
if conn is None:
raise HTTPAPIError(status_code=417, error_detail="cann't connect to mysql",\
notification = "direct", \
log_message= "cann't connect to mysql", \
response = "please check mysqld or user password")
if not os.path.exists(os.path.dirname(options.glb_json_file_name)):
os.mkdir(os.path.dirname(options.glb_json_file_name))
config_file = options.glb_json_file_name % (port)
cmd = r'ps -ef|grep %s|grep -v grep' % (config_file)
result = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
if not result[0] == '':
raise HTTPAPIError(status_code=417, error_detail="this glb is running",\
notification = "direct", \
log_message= "this glb is running", \
response = "please check envirment")
f = open(config_file, 'w')
logging.info("start gbalancer: %s" % (glb_config))
f.write(json.dumps(glb_config, sort_keys=True, indent=4))
f.flush()
f.close()
#self.invokeCommand.run_service_shell(options.start_gbalancer % (config_file, args))
self.invokeCommand.run_check_shell(options.start_gbalancer %
(config_file, args))
time.sleep(1)
glb_proc = self.invokeCommand.run_check_shell(cmd).strip().split('\n')
logging.info("glb_proc: %s" % str(glb_proc))
if len(glb_proc) != 1 or glb_proc[0] == '':
#if len(glb_proc) == 0:
raise HTTPAPIError(status_code=417, error_detail="glb start error",\
notification = "direct", \
log_message= "glb start error", \
response = "please check envirment")
dict = {}
#dict.setdefault("code", '000000')
dict.setdefault("message", "start gbalancer successful!")
self.finish(dict)