def _send_request(text):
try:
response = requests.get(
url="https://westus.api.cognitive.microsoft.com/luis/v2.0/apps/" +
config.microsoft_luis_app_key(),
params={
"staging": "true",
"verbose": "true",
"timezoneOffset": "0",
"subscription-key": config.microsoft_luis_subscription_key(),
"q": text,
},
)
log.debug(f'Response HTTP Status Code: {response.status_code}')
log.debug(f'Response HTTP Response Body: {response.content}')
if response.status_code != 200:
log.error(
f'HTTP Request NOK ({response.status_code}), Body: {response.content}'
)
return json.loads(response.content)
except requests.exceptions.RequestException:
log.error('HTTP Request failed')
def parseConf():
task_cfg = ConfigParser()
try:
task_cfg.read(DEFAULT['realtime_task_config'])
except ParsingError, e:
log.error('配置文件格式错误[%s]'%str(e))
return
def _send_request(audio_file_path):
try:
response = requests.post(
url=
"https://westeurope.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1",
params={
"format": "detailed",
"cid": config.microsoft_speech_endpoint_id(),
},
headers={
"Ocp-Apim-Subscription-Key":
config.microsoft_speech_subscription_key(),
"Content-Type":
"audio/wav; codecs=audio/pcm;",
},
data=open(audio_file_path, 'rb'),
)
log.debug(f'Response HTTP Status Code: {response.status_code}')
log.debug(f'Response HTTP Response Body: {response.content}')
if response.status_code != 200:
log.error(
f'HTTP Request NOK ({response.status_code}), Body: {response.content}'
)
return json.loads(response.content)
except requests.exceptions.RequestException:
log.error('HTTP Request failed')
def clientConnectionFailed(self, connector, reason):
self.defer.errback(
Exception(
json.dumps({
'h': self.host,
'o': str(reason.value),
's': 256
})))
log.error('连接到远程失败:%s' % reason)
def _request(self, url, params, data=None):
_url = f'{self._host}/{url}'
try:
if data:
res = requests.post(_url, params, data)
else:
res = requests.get(_url, params)
except Exception as e:
log.error(f'Connect to {_url} failed, cause by: {e}')
return res
def upload_error_handler(e, original_filename, upload_type):
log.error(f'Remove file due to an error: {original_filename}')
os.remove(original_filename)
log.error('An exception occurred: %r', e)
if upload_type == CSV_FILE:
error_response(f"Currenty only ','-separated files are supported")
if upload_type == ZIP_FILE:
error_response(f"Zip is empty. It only supports: {FILTER_OPTION_LIST}")
error_response(f"Wrong data format")
def sendAll():
"""
purpose:发送alert表中状态是为发送的消息
"""
try:
conn = mdb.connect(host=APOLLOANA_DB['host'], port=APOLLOANA_DB['port'], db=APOLLOANA_DB['db'],
user=APOLLOANA_DB['user'], passwd=APOLLOANA_DB['passwd'], charset='utf8')
except Exception, e:
log.error(e.message)
return
def parseConf():
'''解析任务和主机列表配置文件
'''
host_cfg = ConfigParser()
task_cfg = ConfigParser()
try:
host_cfg.read(DEFAULT['host_file_config'])
task_cfg.read(DEFAULT['task_file_config'])
except ParsingError, e:
log.error(str(e))
return
def test_best_plan(self):
grid = Grid()
grid.init_generate_grid()
grid.show_detail()
result = grid.swap_by_strategy(StrategyType.HIGH_ORDER_FIRST)
if len(result) == 0:
log.error('no swap')
for i in result:
log.info('swap %s' % i)
def parseConf():
'''解析任务和主机列表配置文件
'''
host_cfg = ConfigParser()
task_cfg = ConfigParser()
try:
host_cfg.read(DEFAULT['host_file_config'])
task_cfg.read(DEFAULT['task_file_config'])
except ParsingError, e:
log.error(str(e))
return
def parseCrontab(config):
"""
purpose:解析apollo crontab的配置文件
return:
"""
new_cron_jobs = {}
parser = ConfigParser()
try:
parser.read(config)
except ParsingError, e:
log.error(str(e))
return new_cron_jobs, False
async def on_ready():
print("Bot connected!\n")
print("Logged in as:\n{}/{}#{}\n".format(bot.user.id, bot.user.name,
bot.user.discriminator))
await set_default_status()
for extension in extensions:
try:
bot.load_extension(extension)
except Exception as e:
log.error("Failed to load extension {}\n{}: {}".format(
extension,
type(e).__name__, e))
def parseCrontab(config):
"""
purpose:解析apollo crontab的配置文件
return:
"""
new_cron_jobs = {}
parser = ConfigParser()
try:
parser.read(config)
except ParsingError, e:
log.error(str(e))
return new_cron_jobs, False
def getDBInfo(group):
func_name = 'getDBInfo'
global db_group
curtime = time.time()
if db_group.has_key(group) and db_group[group]['timeout'] > curtime:
return db_group[group]
db_parser = ConfigParser()
try:
db_parser.read(DEFAULT['db_conf'])
except ParsingError, e:
send_default(ERR_TPL % (func_name, str(e)))
log.error(str(e))
return None
def __get_var(self, var):
if not self.__config_file:
try:
with open(self.__application_config_path, 'r') as stream:
self.__config_file = yaml.safe_load(stream)
log.info("Config Loaded")
except FileNotFoundError:
log.info("Config not found, using ENV Var")
return os.environ.get(var)
try:
return os.environ.get(var) or self.__config_file[var]
except KeyError:
log.error('Can not find ENV var: %s' % var)
def sendMessage(self, msg):
"""
向远程服务器报告计划任务执行情况
"""
retries = 0
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.setsockopt(socket.SOL_SOCKET, socket.SNDTIMEO, 5)
while 1:
try:
s.connect(DEFAULT['server_listener'])
s.sendall(msg + self._linedelimiter)
except (socket.gaierror, socket.error), e:
log.error(str(e))
retries += 1
if not retries or retries > 2:
break
def sendMessage(self, msg):
"""
向远程服务器报告计划任务执行情况
"""
retries = 0
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#s.setsockopt(socket.SOL_SOCKET, socket.SNDTIMEO, 5)
while 1:
try:
s.connect(DEFAULT['server_listener'])
s.sendall(msg + self._linedelimiter)
except (socket.gaierror, socket.error), e:
log.error(str(e))
retries += 1
if not retries or retries > 2:
break
def cronNotice(self, info):
"""
purpose: 计划任务的返回信息通知
@parma: info -> 返回信息的dict
"""
func_name = 'cronNotice'
try:
conn = mdb.connect(host=RESOURCE_DB['host'],
port=RESOURCE_DB['port'],
db=RESOURCE_DB['db'],
user=RESOURCE_DB['user'],
passwd=RESOURCE_DB['passwd'],
charset='utf8')
except Exception, e:
send_default(ERR_TPL % (func_name, str(e)))
log.error(e.message)
return
def send_notice(media_and_to, subject, body, level=0):
"""
purpose::将要通知的信息写入到通知表中
@parma::
media_and_to(iters): 通知类型和通知人的元组的集合
subject(str): 消息主题
body(str): 通知的消息体
"""
func_name = 'send_notice'
try:
conn = mdb.connect(host=APOLLOANA_DB['host'],
port=APOLLOANA_DB['port'],
db=APOLLOANA_DB['db'],
user=APOLLOANA_DB['user'],
passwd=APOLLOANA_DB['passwd'],
charset='utf8')
except Exception, e:
log.error(e.message)
return
async def on_command_error(error, ctx):
role = read_data_entry(ctx.message.server.id, "mod-role")
if isinstance(error, commands.CommandNotFound):
return
if isinstance(error, not_mod):
await bot.send_message(
ctx.message.channel,
":no_entry: You must have the `{}` role in order to execute this command!"
)
return
if isinstance(error, not_bot_commander):
await bot.send_message(
ctx.message.channel,
":no_entry: You must have the `Breakfast Eater` role in order to execute this command!"
)
return
if isinstance(error, owner_only):
await bot.send_message(
ctx.message.channel,
":no_entry: Sorry, but only the bot owner can execute this command"
)
return
if isinstance(error, not_server_owner):
await bot.send_message(
ctx.message.channel,
":no_entry: Sorry, but only the server owner (`{}`) can execute this command"
)
return
if ctx.message.channel.is_private:
await bot.send_message(
ctx.message.channel,
"An occur occured! Please try excuting this command on a server.")
return
try:
await bot.send_message(ctx.message.channel, error)
except:
pass
log.error("An error occured while executing the {} command: {}".format(
ctx.command.qualified_name, error))
def send_message(msg, address):
# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
while True:
try:
sock.connect(address)
break
except socket.error:
from util.logger import log
log.error("cannot establish connection to socket %s, please open the socket, this will retry in %s seconds" % (str(address), config.socket_retry_seconds))
time.sleep(config.socket_retry_seconds)
sock.close()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# Send data
send_msg(sock, msg)
finally:
sock.close()
def stringReceived(self, data):
commands = data.split(None, 1)
command_name = commands[0]
args = ''
if len(commands) > 1:
args = commands[1]
command = task_list.get(command_name.strip())
do_cmd = '%s %s' % (command, args)
#实时任务未在任务白名单中
if command is None:
result = {'s': -1, 'o': 'invalid command'}
log.error('不合法的远程请求:%s'%data)
self.connectionEnd(result)
else:
log.info('远程请求:%s'%(do_cmd))
#异步执行实时任务
defer = self.factory.getStatusOutput(do_cmd)
defer.addCallback(self.action)
defer.addErrback(self.errorAction)
def main():
if TOKEN == '':
log.error('!! TOKEN Don\'t exist... Check your "setting.ini" file!')
exit()
elif CHANNEL_ID == ('@' or None or ''):
log.error('!! CHANNEL_ID Don\'t exist... Check your "setting.ini" file!')
exit()
updater = Updater(token=TOKEN)
dp = updater.dispatcher
commands = Commands()
add_handlers(dp, commands)
if not updater.job_queue.jobs():
commands.set_alarms(CHANNEL_ID, FREQUENCY.split(" "), updater.job_queue)
updater.start_polling(timeout=60)
log.info("HBhelperBot Idle...")
updater.idle()
def dealData(self, data):
try:
result = json.loads(data)
except:
log.error('收到的数据不符合json格式')
return
if result['s'] == 0:
tp = int(result['tp'])
if 1 == tp:
try:
tmp = json.loads(result['o'])
#如果是普通字符串,就直接输出到日志
if isinstance(tmp, str):
output = tmp
elif isinstance(tmp, dict):
output = ','.join([
'%s|%s' % (key, value)
for key, value in tmp.iteritems()
])
else:
log.error(result['tk'] + ':未知的数据类型')
except:
log.error(result['tk'] + ':返回数据格式与配置格式不相同')
return
elif 2 == tp:
output = result['o']
else:
log.error(result['tk'] + ':未知的返回格式')
return
line = '%d,%s,%s\n' % (result['tm'], result['h'], output)
fobj = file(
'%s/%s.%s' %
(self.log_dir, result['tk'], strftime("%Y-%m-%d")), 'a')
fobj.write(line)
fobj.flush()
fobj.close()
else:
errmsg = '%s上监控脚本执行错误: %s' % (result['h'], result['o'])
log.error(errmsg)
send_im(to='liujun2,cuihua', msg=errmsg)
def get_work_days(days_list):
"""
:param days_list: ['20160101', '20160202']
:return: if single days_list, return 0 or 1 or 2 ,
if multiple days_list, return {"20130101":"2","20130103":"2",,"20130201":"0"}
"""
# url = 'http://apis.baidu.com/xiaogg/holiday/holiday?d=' # baidu
# url = 'http://a.apix.cn/tbip/sinaapp/?d=' # damn it!
url = 'http://www.easybots.cn/api/holiday.php?d='
prefix = ','.join(days_list)
try:
response = requests.get('%s%s' % (url, prefix))
if len(days_list) == 1:
return json.loads(response.text)[days_list[0]]
else:
return json.loads(response.text)
except Exception, e:
log.error(e)
return None
def send_message(msg, address):
# Create a TCP/IP socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
while True:
try:
sock.connect(address)
break
except socket.error:
from util.logger import log
log.error(
"cannot establish connection to socket %s, please open the socket, this will retry in %s seconds"
% (str(address), config.socket_retry_seconds))
time.sleep(config.socket_retry_seconds)
sock.close()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# Send data
send_msg(sock, msg)
finally:
sock.close()
def updateStats(self, info, others):
"""
根据远程命令执行结果更新数据库对应的状态字段
- info 远程agent返回来的数据
- others 要执行更新的数据库的相关的数据库相关信息字典
包括db(数据库名称),tb(表名),keys(更新字段别名对应)
"""
func_name = 'updateStatus'
if others is None:
log.error('没有指定相应的数据库信息来更新字段')
return
retinfo = {}
try:
result = json.loads(info)
except Exception, e:
try:
result = json.loads(info.value.message)
except Exception, e:
send_default(ERR_TPL % (func_name, '返回结果格式不正确'))
log.error(ERR_TPL % (func_name, str(e)))
return
def orange():
user_socket = request.environ.get('wsgi.websocket')
log.info('request %s', request)
log.info('user_socket %s', user_socket)
if user_socket:
user_socket_list.append(user_socket)
log.info('user_socket_list depth %s', len(user_socket_list))
while True:
try:
msg = user_socket.receive()
log.info('received: %s', msg)
for sock in user_socket_list:
try:
sock.send('received: %s' + msg)
except:
continue
except Exception as e:
log.error(e)
if user_socket and user_socket in user_socket_list:
user_socket_list.remove(user_socket)
return ''
def lineReceived(self, line):
try:
info = json.loads(line)
except:
log.error('json字符串%s解析失败')
self.transport.lostConnection()
if 'RESPONSE' == info['req_type']:
if 'CRON' == info['res_type']:
log.info(info['retval'])
self.cronCallBack(info['retval'])
elif 'TRANSMIT' == info['req_type']:
cmdinfo = info['cmdinfo'].split(':', 1)
others = info.get('others', None)
self.transCommand({
'h': cmdinfo[0],
'c': cmdinfo[1],
'a': info['res_type'],
'others': others
})
else:
self.transport.loseConnection()
def dealData(self, data):
try:
result = json.loads(data)
except:
log.error('收到的数据不符合json格式')
return
if result['s'] == 0:
tp = int(result['tp'])
if 1 == tp:
try:
tmp = json.loads(result['o'])
#如果是普通字符串,就直接输出到日志
if isinstance(tmp, str):
output = tmp
elif isinstance(tmp, dict):
output = ','.join(['%s|%s'%(key,value) for key,value in tmp.iteritems()])
else:
log.error(result['tk'] + ':未知的数据类型')
except:
log.error(result['tk'] + ':返回数据格式与配置格式不相同')
return
elif 2 == tp:
output = result['o']
else:
log.error(result['tk'] + ':未知的返回格式')
return
line = '%d,%s,%s\n' % (result['tm'], result['h'], output)
fobj = file('%s/%s.%s'%(self.log_dir, result['tk'], strftime("%Y-%m-%d")),'a')
fobj.write(line)
fobj.flush()
fobj.close()
else:
errmsg = '%s上监控脚本执行错误: %s'%(result['h'], result['o'])
log.error(errmsg)
send_im(to='liujun2,cuihua', msg=errmsg)
def connectionLost(self, reason):
"""当连接失败时停止发送心跳包 """
self.factory._paused = True
log.error('connection lost: %s' % reason)
NetstringReceiver.connectionLost(self,reason)
self.stopHeartbeat()
def clientConnectionFailed(self, connector, reason):
log.error('connection failed: %s' % reason)
self._paused = True
ReconnectingClientFactory.clientConnectionFailed(self, connector, reason)
def _timeoutConnection(self):
log.error('心跳超时')
#self.transport.loseConnection()
self.factory._paused = True
purpose:发送alert表中状态是为发送的消息
"""
try:
conn = mdb.connect(host=APOLLOANA_DB['host'], port=APOLLOANA_DB['port'], db=APOLLOANA_DB['db'],
user=APOLLOANA_DB['user'], passwd=APOLLOANA_DB['passwd'], charset='utf8')
except Exception, e:
log.error(e.message)
return
now = int(time.time())
sql = "select alert_id,mediatype_id,sendto,subject,msg,retries,level from ana_report_alerts where status=0 and clock>%d" % (now-600)
cur = conn.cursor(mdb.cursors.DictCursor)
try:
cur.execute(sql)
except Exception, e:
send_im(to='liujun2,cuihua',msg=e.message)
log.error(e.message)
return
for alert in cur.fetchall():
if MEDIA['im'] == alert['mediatype_id']:
ret = send_im(to=alert['sendto'], msg=alert['msg'])
elif MEDIA['email'] == alert['mediatype_id']:
level = 1
if 1 == alert['level']:
level = 3
ret = send_email( to=alert['sendto'],
subject=alert['subject'], body=alert['msg'].replace('\n', '<br/>'),
level=level)
elif MEDIA['sms'] == alert['mediatype_id']:
ret = send_sms(phones=alert['sendto'],content=alert['msg'])
else:
hostname = getHostname()
conf_jobs = [(sec, dict(parser.items(sec))) for sec in parser.sections()]
for sec in parser.sections():
if not sec.isdigit(): continue
job = dict(parser.items(sec))
if hostname in [name.strip() for name in job['hostlist'].split(',')]:
fields = job['task_interval'].split()
try:
trigger = CronTrigger(minute=fields[0],
hour=fields[1],
day=fields[2],
month=fields[3],
day_of_week=fields[4])
except Exception, e:
log.error(e.message)
continue
#print trigger.get_next_fire_time(datetime.now())
new_cron_jobs[sec] = {
'name': job['task_name'],
'args': [unquote(job['task_content']).strip(), sec],
'trigger': trigger
}
return new_cron_jobs, True
class ApolloCrond(object):
""" 任务调度类 """
_linedelimiter = '\r\n'
code_string = {
def exceptions(e):
tb = traceback.format_exc()
log.error('%s %s %s %s 5xx INTERNAL SERVER ERROR\n%s', request.remote_addr,
request.method, request.scheme, request.full_path, tb)
return e.status_code
def error_handler(bot, update, err):
if update:
log.error('!! Update %s :: %s' % (update, err))
