def check_salt_minion_fun():
check_salt_minion = check_status.objects.filter(program='check_salt_minion').first()
if check_salt_minion.status == 1:
sapi = SaltAPI(
url = settings.SALT_API['url'],
username = settings.SALT_API['user'],
password = settings.SALT_API['password']
)
sapi_glb = SaltAPI(
url = settings.SALT_API['url_glb'],
username = settings.SALT_API['user'],
password = settings.SALT_API['password']
)
minionsup, minionsdown= sapi.MinionStatus()
minionsup_glb, minionsdown_glb= sapi_glb.MinionStatus()
minionsdown_glb = [id for id in minionsdown_glb if 'SC' in id]
else:
minionsdown = []
minionsdown_glb = []
if len(minionsdown) != 0:
send_mail(get_mail_list('check_salt_minion'),'Attention: salt_minion','Minion Down:'+ '\n\t' +'\n\t'.join(minionsdown))
if len(minionsdown_glb) != 0:
send_mail(admin_mail_addr,'Attention: 市场部服务器故障','Minion Down:'+ '\n\t' +'\n\t'.join(minionsdown_glb))
end_time['check_salt_minion'] = time()
def EditConfig(request):
username = request.user.username
try:
role = request.user.userprofile.role
except:
role = 'none'
clientip = request.META['REMOTE_ADDR']
if role != 'sa':
return HttpResponseForbidden('你没有权限进行编辑。')
data = json.loads(request.body)
logger.info('%s is requesting. %s' % (clientip, request.get_full_path()))
if data['act'] == 'getfiles':
ret = svn_config(data).getFiles()
if ret['code'] == 200:
return HttpResponse(json.dumps(ret))
elif ret['code'] == 500:
return HttpResponseServerError(ret['response'])
elif data['act'] == 'commitfiles':
data['commit'] = "%s %s %s commit svn config files: %s %s %s" % (time(
), clientip, username, data['envir'], data['project'], data['svn_id'])
ret = svn_config(data).commitFiles()
if ret['code'] == 200:
return HttpResponse(ret['response'])
elif ret['code'] == 500:
return HttpResponseServerError(ret['response'])
def check_services_fun():
check_services = check_status.objects.filter(program='check_services').first()
if check_services.status == 1:
content = check_tomcat()
if content != "":
send_mail(get_mail_list('check_services'),'tomcat报警',content,format='html')
end_time['check_services'] = time()
def check_salt_intrm():
try:
ret = requests.head(salt_intrm)
except requests.exceptions.ConnectionError:
logger.error('%s server is down or refused to connect.' %salt_intrm)
check_salt_intrm_false(salt_intrm_id, salt_intrm, 'connection refused')
except:
logger.error('%s unkown error, pls check the script.' %salt_intrm)
check_salt_intrm_false(salt_intrm_id, salt_intrm, 'unkown error, pls check the script')
else:
if ret.status_code == 200:
logger.info('%s is running.' %salt_intrm)
else:
logger.error('%s check failed. Return status: %s' %(salt_intrm, ret.status_code))
check_salt_intrm_false(salt_intrm_id, salt_intrm, ret.status_code)
finally:
end_time['check_salt_intrm'] = time()
def check_services_fun():
content = check_services()
#check_services = check_status.objects.filter(program='check_services').first()
#if check_services.status == 1:
# content = check_services()
# if content != "":
# send_mail(get_mail_list('check_services'),'tomcat报警',content,format='html')
if content != "":
message = {}
url = settings.TG_API['url'] + '/sendMessage'
message['chat_id'] = settings.TG_API['chat_id']['salt_minion_alert']
message['text'] = content
#message['parse_mode']='HTML'
try:
ret = requests.post(url, data=message, timeout=3)
except:
print 'send message failed.'
end_time['check_services'] = time()
def check_salt_intrm_false(salt_intrm_id, salt_intrm, status_code):
result = sapi.checkMinion(
tgt = salt_intrm_id,
expr_form = 'glob',
)
if result[0].has_key(salt_intrm_id):
logger.info('%s status is %s.' %(salt_intrm_id, result[0][salt_intrm_id]))
salt_intrm_id_status = result[0][salt_intrm_id]
else:
logger.error('%s status is down.' %salt_intrm_id)
salt_intrm_id_status = 'not return'
mail_content = (
"-------------------------------------------------\n"
"检测时间: %s\n"
"检测服务: %s\n"
"检测状态: %s\n"
"%s: %s\n"
"-------------------------------------------------"
)%(time(), salt_intrm, status_code, salt_intrm_id, salt_intrm_id_status)
send_mail(admin_mail_addr, 'Attention: salt_intrm check failed!', mail_content)
def Execute(self):
#执行升级，比对代码和回退
if self.__data['act'] == 'rollback':
#执行回退的api
self.__data['exe_api'] = self.__data[
'api'] + '?project=%s&ip_addr=%s&action=%s&restart=%s&backup_file=%s&envir=%s' % (
self.__data['project'], self.__data['ip_addr'],
self.__data['act'], self.__data['restart'],
self.__data['backup_file'], self.__data['envir'])
else:
#执行升级和比对代码的api
self.__data['exe_api'] = self.__data[
'api'] + '?project=%s&ip_addr=%s&action=%s&restart=%s&svn_id=%s&envir=%s' % (
self.__data['project'], self.__data['ip_addr'],
self.__data['act'], self.__data['restart'],
self.__data['svn_id'], self.__data['envir'])
try:
self.__data['op_time'] = time()
#执行get请求
ret = requests.get(self.__data['exe_api'],
timeout=self.__data['timeout'])
except requests.exceptions.ReadTimeout:
logger.error("%s, ReadTimeout! timeout: %s s" %
(self.__data['exe_api'], self.__data['timeout']))
self.__data['result'] = '%s ReadTimeout' % self.__data['exe_api']
self.__data['op_status'] = 0
self.__data['com_time'] = time()
except requests.exceptions.ConnectionError:
logger.error("%s, ConnectionError!" % self.__data['exe_api'])
self.__data[
'result'] = '%s ConnectionError' % self.__data['exe_api']
self.__data['op_status'] = 0
self.__data['com_time'] = time()
except:
logger.error("%s, UnknownError!" % (self.__data['exe_api']))
self.__data['result'] = '%s UnknownError' % self.__data['exe_api']
self.__data['op_status'] = 0
self.__data['com_time'] = time()
else:
logger.info("%s, %s success!" %
(self.__data['exe_api'], self.__data['act']))
self.__data['result'] = ret.content
self.__data['op_status'] = 1
self.__data['com_time'] = time()
finally:
try:
self.__data['result'] = json.loads(self.__data['result'])
self.__data['info'] = self.__data['result'][self.__data['act']]
except ValueError:
logger.error('%s result is string: %s' %
(self.__data['act'], self.__data['result']))
self.__data['info'] = self.__data['result']
except:
logger.error('%s result doesn\'t has key %s: %s' %
(self.__data['act'], self.__data['act'],
self.__data['result']))
self.__data['info'] = self.__data['result']
if self.__data['act'] == 'deploy':
try:
self.__data['backup_file'] = self.__data['result'][
'backup_file']
except:
logger.error(
'deploy result doesn\'t has key backup_file: %s' %
self.__data['result'])
self.__data['backup_file'] = 'Null'
elif self.__data['act'] == 'diff':
self.__data['backup_file'] = 'Null'
try:
op_record = op_history(
svn_id=self.__data['svn_id'],
project=self.__data['project'],
ip_addr=self.__data['ip_addr'],
act=self.__data['act'],
op_user=self.__data['username'],
op_ip_addr=self.__data['clientip'],
op_status=self.__data['op_status'],
envir=self.__data['envir'],
info=self.__data['info'],
op_time=self.__data['op_time'],
com_time=self.__data['com_time'],
backup_file=self.__data['backup_file'],
)
op_record.save()
except:
self.__data['result'] = 'InsertRecordError'
self.__data['op_status'] = 0
logger.error('insert into upgrade_op_history failed: %s' %
self.__data)
finally:
return self.__data
logger.error('%s server is down or refused to connect.' %salt_intrm)
check_salt_intrm_false(salt_intrm_id, salt_intrm, 'connection refused')
except:
logger.error('%s unkown error, pls check the script.' %salt_intrm)
check_salt_intrm_false(salt_intrm_id, salt_intrm, 'unkown error, pls check the script')
else:
if ret.status_code == 200:
logger.info('%s is running.' %salt_intrm)
else:
logger.error('%s check failed. Return status: %s' %(salt_intrm, ret.status_code))
check_salt_intrm_false(salt_intrm_id, salt_intrm, ret.status_code)
finally:
end_time['check_salt_intrm'] = time()
if __name__ == '__main__':
start_time = time()
end_time = multiprocessing.Manager().dict()
if not check_server_status():
mail_content = (
"-------------------------------------------------\n"
"检测时间: %s\n"
"检测服务: %s\n"
"检测状态: %s\n"
"-------------------------------------------------\n"
)%(time(), server, '失败')
os.system('nohup python %s/manage.py runserver 0.0.0.0:5000 &' %basedir)
#send_mail(admin_mail_addr, 'Attention: django_server is down!', mail_content)
mail_content = mail_content + (
"尝试重启服务......\n"
"重启时间: %s\n"
)%time()
minionsup_glb, minionsdown_glb = sapi_glb.MinionStatus()
minionsdown_glb = [id for id in minionsdown_glb if 'SC' in id]
else:
minionsdown = []
minionsdown_glb = []
if len(minionsdown) != 0:
send_mail(get_mail_list('check_salt_minion'), 'Attention: salt_minion',
'Minion Down:' + '\n\t' + '\n\t'.join(minionsdown))
if len(minionsdown_glb) != 0:
send_mail(admin_mail_addr, 'Attention: 市场部服务器故障',
'Minion Down:' + '\n\t' + '\n\t'.join(minionsdown_glb))
end_time['check_salt_minion'] = time()
if __name__ == '__main__':
start_time = time()
end_time = multiprocessing.Manager().dict()
#if not check_server_status():
# mail_content = (
# "-------------------------------------------------\n"
# "检测时间: %s\n"
# "检测服务: %s\n"
# "检测状态: %s\n"
# "-------------------------------------------------\n"
# )%(time(), server, '失败')
# os.system('nohup python %s/manage.py runserver 0.0.0.0:5000 &' %basedir)
# #send_mail(admin_mail_addr, 'Attention: django_server is down!', mail_content)
# mail_content = mail_content + (
# "尝试重启服务......\n"
# "重启时间: %s\n"
# )%time()