def getNetdataConfig(mainConfig, sectionName):
"""
Get netdata server config and also check backend configuration
Input: Config: mainConfig : All configuration
str sectionName : Section Name of that specific config
Returns: boolean: True - success in getting config and all parameters defined correctly.
False - Failure in retrieving one or another configuration parameters.
dict: {} - if boolean return is False, always empty as failed to retrieve.
otherwise dictionary of all parsed configuration parameters
"""
# TODO look at netdata filters, and ignore which are not published to Database
hostname = mainConfig.get(sectionName, 'hostname')
outDict = {}
with tempfile.NamedTemporaryFile(delete=False) as fd:
try:
tmpConf = getDataFromUrl("%s/netdata.conf" % hostname)
for line in tmpConf.read().splitlines():
fd.write(line.replace('\t', '') + "\n")
except urllib2.URLError as ex:
print 'Received URLError %s for %s. Checking if config file is present locally' % (
ex, hostname)
if os.path.isfile("%s.conf" % sectionName):
with open("%s.conf" % sectionName, 'r') as fdConf:
for line in fdConf:
fd.write(line.replace('\t', '') + "\n")
else:
print 'Config file is also not present. Skipping this node %s check' % hostname
return False, {}
tempfileName = fd.name
hostConfig = getConfig([tempfileName])
if not hostConfig.has_section('backend'):
print 'Hostname %s Netdata server is not configured to publish anything to any backend' % hostname
print '* Skipping this node check.'
return False, {}
if not hostConfig.has_option('backend', 'enabled'):
print 'Hostname %s Netdata does not have enabled option in backend configuration' % hostname
print '* Skipping this node check.'
return False, {}
if not hostConfig.has_option('backend', 'destination'):
print 'Hostname %s Netdata is configured to send metrics but destination is not set.' % hostname
print '* Skipping this node check.'
return False, {}
if not hostConfig.has_option('backend', 'prefix'):
outDict['prefix'] = 'netdata'
for optionKey in hostConfig.options('backend'):
outDict[optionKey] = hostConfig.get('backend', optionKey)
print outDict
# Make boolean from send names instead of ids
if 'send names instead of ids' in outDict:
if outDict['send names instead of ids'] == 'yes':
outDict['dimensionids'] = True
elif outDict['send names instead of ids'] == 'no':
outDict['dimensionids'] = False
else:
outDict['dimensionids'] = True
else:
outDict['dimensionids'] = False
outDict['send names instead of ids'] = 'no'
return True, outDict
def get_cron_task(self):
if self.pkg_conf is None:
self.pkg_conf = get_pkg_config(self.package_id, self.package_path)
self.pkg_cron = common.getConfig(self.pkg_conf, 'crontab')
if self.pkg_cron == "":
return 0
return 0
def __init__(self):
super(test_ALI_5, self).__init__("test_ALI_5")
common.sensemediaTestBase.setlogger(self, __name__)
self.testid="ALI-5"
#下文的self.tpid
self.testplanid=""
#建议获取最新的buildname(),下文中的self.build_name
self.buildname=""
#ｅｇ:"2019-01-18 14:55"
self.timestamp=time.strftime("%Y-%m-%d %H:%m")
self.testcasestarttime=time.time()
self.execduration=""
#获取TestLink 该用例的一些信息
self.testlink_url = common.getConfig("url", "testlinkurl")
self.testlink_key = common.getConfig("url", "testlinkkey")
self.project_id = common.getConfig("url", "project_id")
self.project_name = common.getConfig("url", "project_name")
self.logger.info("project name is %s "% self.project_name)
self.testPlanName = common.getConfig("url", "testPlanName")
def getDBlocation(retvar):
custdb = addon.getSetting('customdbfolder') == 'true'
old_dbpath = xbmc.translatePath(getConfig('old_dbfolder')).decode('utf-8')
cur_dbpath = dbpath
if not old_dbpath:
old_dbpath = cur_dbpath
if custdb:
cur_dbpath = xbmc.translatePath(
addon.getSetting('dbfolder')).decode('utf-8')
else:
addon.setSetting('dbfolder', dbpath)
orgDBfile = {
'tv': os.path.join(dbpath, 'tv.db'),
'movie': os.path.join(dbpath, 'movies.db')
}
oldDBfile = {
'tv': os.path.join(old_dbpath, 'tv.db'),
'movie': os.path.join(old_dbpath, 'movies.db')
}
DBfile = {
'tv': os.path.join(cur_dbpath, 'tv.db'),
'movie': os.path.join(cur_dbpath, 'movies.db')
}
if old_dbpath != cur_dbpath:
Log('DBPath changed')
if xbmcvfs.exists(oldDBfile['tv']) and xbmcvfs.exists(
oldDBfile['movie']):
if not xbmcvfs.exists(cur_dbpath):
xbmcvfs.mkdir(cur_dbpath)
if not xbmcvfs.exists(DBfile['tv']) or not xbmcvfs.exists(
DBfile['movie']):
copyDB(oldDBfile, DBfile)
writeConfig('old_dbfolder', cur_dbpath)
if custdb:
org_fileacc = int(
xbmcvfs.Stat(orgDBfile['tv']).st_mtime() +
xbmcvfs.Stat(orgDBfile['movie']).st_mtime())
cur_fileacc = int(
xbmcvfs.Stat(DBfile['tv']).st_mtime() +
xbmcvfs.Stat(DBfile['movie']).st_mtime())
if org_fileacc > cur_fileacc:
copyDB(orgDBfile, DBfile, True)
return DBfile[retvar]
def get_config(self, config_file):
clear_conf = common.getConfig(config_file, 'clear_file')
# 处理以#开头的行
regex = r"\s*#"
conf_arr = clear_conf.splitlines()
real_conf = []
for line in conf_arr:
ret = re.match(regex, line)
if ret is None:
# 处理#注释在行尾的情况
reg_2 = r"^((\"[^\"]*\"|'[^']*'|[^'\"#])*)(#.*)$"
ret = re.match(reg_2, line)
if ret is not None:
conf_line = ret.group(1)
else:
conf_line = line
conf = re.split(r'\s+', conf_line)
if len(conf) < 5:
continue
real_conf.append(conf)
return real_conf
def main():
parser = OptionParser()
parser.add_option("-b", "--bot", dest="bot", default="me", help="IDREF of bot in config file")
parser.add_option("-p", "--pubsub", dest="pubsub", default="tune", help="IDREF of pubsub in config file")
(options, args) = parser.parse_args()
config = common.getConfig()
try:
bot_jid=xmpp.protocol.JID(config("users/user[@id='%s']/jid/text()"%(options.bot))[0].nodeValue + '/tunepub/mpd')
bot_passwd = config("users/user[@id='%s']/password/text()"%(options.bot))[0].nodeValue
pubsub_service = config("pubsub/node[@id='%s']/service/text()"%(options.pubsub))[0].nodeValue
pubsub_id = config("pubsub/node[@id='%s']/id/text()"%(options.pubsub))[0].nodeValue
except:
print 'Could not find proper nodes in config file (Looked for bot \'%s\' and pubsub node \'%s\')'%(options.bot, options.pubsub)
sys.exit()
c = None
last_song = ''
client = xmpp.Client(bot_jid.getDomain(), debug=[])
client.connect()
client.auth(bot_jid.getNode(), bot_passwd)
while True:
if c == None:
try:
c = mpdclient.MpdController(host="localhost", port=6600)
except:
c = None
if c == None:
mpd_song = False
else:
try:
mpd_song = c.getCurrentSong()
except:
c = None
mpd_song = False
try:
if mpd_song == False:
raise Exception
status = c.getStatus()
title = mpd_song.title
length = status.totalTime
artist = mpd_song.artist
source = mpd_song.path
if artist == '' and title.count(' - ') > 0:
parts = title.split(' - ')
artist = parts[0]
title = parts[1]
if last_song != title:
escape = xmpp.simplexml.XMLescape
node_id = md5.md5(time.time().__str__()).hexdigest()
node_xml = "<iq type='set' from='%s' to='%s' id='%s'><pubsub xmlns='http://jabber.org/protocol/pubsub'><publish node='%s'><item id='current'><tune xmlns='http://jabber.org/protocol/tune'><artist>%s</artist><title>%s</title><source>%s</source><length>%s</length></tune></item></publish></pubsub></iq>"%(bot_jid,pubsub_service,node_id,pubsub_id,escape(artist),escape(title),escape(source),length)
tune_node = xmpp.simplexml.XML2Node(node_xml)
tune_iq = xmpp.protocol.Iq(node=tune_node)
except:
title = 'Stopped'
if last_song != title:
node_id = md5.md5(time.time().__str__()).hexdigest()
node_xml = "<iq type='set' from='%s' to='%s' id='%s'><pubsub xmlns='http://jabber.org/protocol/pubsub'><publish node='%s'><item id='current'><tune xmlns='http://jabber.org/protocol/tune' /></item></publish></pubsub></iq>"%(bot_jid,pubsub_service,node_id,pubsub_id)
tune_node = xmpp.simplexml.XML2Node(node_xml)
tune_iq = xmpp.protocol.Iq(node=tune_node)
if last_song != title:
client.send(tune_iq)
last_song = title
try:
time.sleep(5)
except KeyboardInterrupt:
sys.exit()
def main():
"""
Main execution. For all config specified netdata hosts, prepare grafana dashboard template.
Input: None
Returns: None
"""
# TODO. Allow to specify configuration location.
allConfigs = {"HOSTS": {}}
mainConfig = getConfig(['netdata-grafana-hosts.conf'])
allConfigs['backend'] = mainConfig.get('global', 'backend')
allConfigs['grafanaUrl'] = mainConfig.get('global', 'grafanaUrl')
if allConfigs['backend'] == 'opentsdb':
allConfigs['opentsdb'] = {
"datasource": mainConfig.get('opentsdb', 'datasource'),
"order": checkOrderConfig(mainConfig, 'opentsdb'),
"skipOthers": checkSkipOthers(mainConfig, 'opentsdb')
}
allConfigs['opentsdb']['title'] = mainConfig.get('opentsdb', 'title')
allConfigs['opentsdb']['description'] = mainConfig.get(
'opentsdb', 'description')
allConfigs['opentsdb']['dimensionids'] = mainConfig.getboolean(
'opentsdb', 'dimensionids')
allConfigs['opentsdb']['prefix'] = mainConfig.get('opentsdb', 'prefix')
allConfigs['opentsdb']['tags'] = getTags(mainConfig, 'opentsdb')
allConfigs['opentsdb']['customfilters'] = json.loads(
mainConfig.get('opentsdb', 'customfilters'))
# get customFirstRow and customLastRow
allConfigs['opentsdb']['customFirstRow'] = getValFromConfig(
mainConfig, 'opentsdb', 'customFirstRow')
allConfigs['opentsdb']['customLastRow'] = getValFromConfig(
mainConfig, 'opentsdb', 'customLastRow')
for sectionName in mainConfig.sections():
if sectionName in ['global', 'opentsdb']:
continue
# check if mandatory options are in place
if not (mainConfig.has_option(sectionName, 'hostname')
and mainConfig.get(sectionName, 'hostname')):
print 'In section %s hostname is not defined. It is mandatory to define full url' % sectionName
print '* Skipping this node check.'
continue
if allConfigs['backend'] == 'graphite':
if not (mainConfig.has_option(sectionName, 'datasource')
and mainConfig.get(sectionName, 'datasource')):
print 'In section %s dataspirce is not defined. It is mandatory to define datasource' % sectionName
print '* Skipping this node check.'
continue
configSuccess, config = getNetdataConfig(mainConfig, sectionName)
if not configSuccess:
config[
'SKIP_NODE'] = False # This is not looked in case of graphite. TODO
config['tags'] = getTags(mainConfig, allConfigs['backend'])
if allConfigs['backend'] == 'graphite':
# This is relevant only for graphite
config['datasource'] = mainConfig.get(sectionName, 'datasource')
config['order'] = checkOrderConfig(mainConfig, sectionName)
config['skipOthers'] = checkSkipOthers(mainConfig, sectionName)
config['hostname'] = mainConfig.get(sectionName, 'hostname')
config['section'] = sectionName
# get customFirstRow and customLastRow
config['customFirstRow'] = getValFromConfig(mainConfig, sectionName,
'customFirstRow')
config['customLastRow'] = getValFromConfig(mainConfig, sectionName,
'customLastRow')
allConfigs["HOSTS"][config['hostname']] = config
print allConfigs
# Now send allConfigs to a specific backend preparator.
if allConfigs['backend'] == 'graphite':
graphiteDashboard(allConfigs)
elif allConfigs['backend'] == 'opentsdb':
opentsdbDashboard(allConfigs)
else:
print 'Unknown backend type... Exiting'
#!/usr/bin/python
import sys,os,xmpp,common
from time import time
from optparse import OptionParser
from pysqlite2 import dbapi2 as sqlite
parser = OptionParser()
parser.add_option("-b", "--bot", dest="bot", default="bot", help="IDREF of bot in config file")
(options, args) = parser.parse_args()
config = common.getConfig()
try:
bot_jid=xmpp.protocol.JID(config("//user[@id='%s']/jid/text()"%(options.bot))[0].nodeValue + '/presencebot')
bot_passwd = config("//user[@id='%s']/password/text()"%(options.bot))[0].nodeValue
except:
print 'Could not find proper nodes in config file (Looked for bot \'%s\')'%(options.bot)
sys.exit();
presence_db = sqlite.connect(config("//db/text()")[0].nodeValue,isolation_level=None)
tune_db = sqlite.connect(config("//db/text()")[0].nodeValue,isolation_level=None)
def main():
cur = presence_db.cursor()
# cur.execute('DELETE FROM presence')
client = xmpp.Client(bot_jid.getDomain(), debug=[])
client.connect()
client.auth(bot_jid.getNode(), bot_passwd, bot_jid.getResource())
client.sendInitPresence()
client.RegisterHandler('presence', do_presence)
client.RegisterHandler('message', do_message)
while step(client):
def reportStatus(self,
pkg_id,
pkg_conf_path,
inst_info,
status,
err_info={}):
reportInfo = {}
reportInfo['dims'] = {}
reportInfo['vals'] = {}
reportInfo['dims']['process.process.package_id'] = pkg_id
reportInfo['dims']['process.process.version_id'] = ""
reportInfo['dims']['process.process.install_path'] = inst_info[
'installPath']
reportInfo['vals']['process.process.package_status'] = status
if not err_info.get('err_proc') and not err_info.get('err_port'):
reportInfo['vals']['process.process.alert_status'] = 0
else:
reportInfo['vals']['process.process.alert_status'] = 1
conf_file = os.path.join(pkg_conf_path, 'package.conf.yaml')
proc_config = common.getConfig(conf_file, "proc_list")
port_config = common.getConfig(conf_file, "port_list")
proc_list = {}
for proc in proc_config:
proc_list[proc['proc_name']] = proc
normal_proc_list = []
proc_num_list = self.pkgCom.getProcNum()
normal_proc_str = ""
for proc_name in err_info.get('ok_proc', []):
num_min = proc_list[proc_name].get('proc_num_min', 0)
num_max = proc_list[proc_name].get('proc_num_max', 0)
proc_str = "%s:%s,%s|%s" % (proc_name, num_min, num_max,
proc_num_list[proc_name])
normal_proc_list.append(proc_str)
normal_proc_str = '##'.join(normal_proc_list)
abnormal_proc_list = []
abnormal_proc_str = ""
for proc_name in err_info.get('err_proc', []):
num_min = proc_list[proc_name].get('proc_num_min', 0)
num_max = proc_list[proc_name].get('proc_num_max', 0)
proc_str = "%s:%s,%s|%s" % (proc_name, num_min, num_max,
proc_num_list[proc_name])
abnormal_proc_list.append(proc_str)
abnormal_proc_str = '##'.join(abnormal_proc_list)
normal_port_list = []
normal_port_str = ""
for port in err_info.get('ok_port', []):
normal_port_list.append(port)
normal_port_str = "##".join(map(str, normal_port_list))
abnormal_port_list = []
abnormal_port_str = ""
for port in err_info.get('err_port', []):
abnormal_port_list.append(port)
abnormal_port_str = "##".join(map(str, abnormal_port_list))
reportInfo['vals'][
'process.process.normal_processes'] = normal_proc_str
reportInfo['vals'][
'process.process.abnormal_processes'] = abnormal_proc_str
reportInfo['vals']['process.process.normal_ports'] = normal_port_str
reportInfo['vals'][
'process.process.abnormal_ports'] = abnormal_port_str
if report:
ret, msg = report(data_id=3000,
dims=reportInfo['dims'],
vals=reportInfo['vals'])
def monitorPkg(self, pkg_info):
import random
gevent.sleep(random.randint(0, 15))
pkg_id, pkg_conf_path, pkg_install_path = pkg_info[
'packageId'], pkg_info['confPath'], pkg_info['installPath']
# 初始化log配置
st = time.time()
mLog = self.pkgCom.configLog(pkg_install_path, 'monitor',
self.log_level)
mLog.info('check process start: %s' % pkg_install_path)
# 读取实例信息
inst_conf_file = os.path.join(pkg_conf_path, 'instance.conf.yaml')
with open(inst_conf_file, 'r') as fp:
conf_info = yaml.load(fp)
inst_info = conf_info.pop()
# 检查包是否已启动
status_conf_file = os.path.join(pkg_conf_path, 'package.status')
if not os.path.isfile(status_conf_file):
return 0, 'ok'
with open(status_conf_file, 'r') as fp:
status_info = yaml.load(fp)
start_status = status_info['status']
if start_status == 'stopped':
self.reportStatus(pkg_id,
pkg_conf_path,
inst_info,
status=start_status)
mLog.info('package status is stopped: %s %s' %
(pkg_info['installPath'], round(time.time() - st, 4)))
return 0, 'ok'
# 获得文件锁，准备检查。且快速失败，因为进程检查本身就是1分钟执行一次的
ret = self.pkgCom.getLock(pkg_conf_path, timeout=10)
if not ret:
mLog.error("get lock error")
return 2008, "get lock error"
# 根据包配置信息检查包的进程状态
conf_file = os.path.join(pkg_conf_path, 'package.conf.yaml')
err_proc, ok_proc = self.pkgCom.checkProcStatus(
conf_file, install_path=inst_info.get('installPath'))
err_port, ok_port = self.pkgCom.checkPort(
conf_file, install_path=inst_info.get('installPath'))
proc_config = common.getConfig(conf_file, "proc_guard")
port_config = common.getConfig(conf_file, "port_guard")
err_info = {
'err_proc': err_proc,
'ok_proc': ok_proc,
'err_port': err_port,
'ok_port': ok_port
}
self.reportStatus(pkg_id,
pkg_conf_path,
inst_info,
status=start_status,
err_info=err_info)
code = 0
msg = 'ok'
err_msg = ""
if err_proc:
err_msg += ",error process:" + ",".join(err_proc)
if err_port:
err_msg += ",error port:" + ",".join(map(str, err_port))
# 包操作对象
op = pkgOp(self.APP_BASE, inst_info['installPath'])
if (err_proc and proc_config
== 'stopStart') or (err_port and port_config == 'stopStart'):
msg = "process error,monitor run stopStart:" + err_msg
mLog.info(msg)
code, msg = op.stopStart(inst_info['packageId'],
inst_info['installPath'])
elif (err_proc
and proc_config == 'custom') or (err_port
and port_config == 'custom'):
msg = "process error,monitor run custom script:" + err_msg
mLog.info(msg)
code, msg = op.resolve(inst_info['packageId'],
inst_info['installPath'])
elif err_proc or err_port:
msg = "process error,do nothing:" + err_msg
mLog.info(msg)
# 解锁
self.pkgCom.unLock(pkg_conf_path)
mLog.info('check process end: %s %s' %
(pkg_info['installPath'], round(time.time() - st, 4)))
return code, msg
self.log.rotateLogs()
#ротируем директории рабочих
worker.createEmptyDirs(*self.worker_dirs)
#запускаем рабочих
self.workers_cnt = workers_cnt
self.putWorkersInOrder()
def handleSIGTERM(self, sig_num, frame):
self.log.write('OK. FINISHING')
#очищаю список задач
with open(self.tasks_file, 'w') as output:
while not self.tasks_queue.empty():
output.write(self.tasks_queue.get() + '\n')
#убиваем всех рабочих
workers_cnt = self.workers_cnt
self.workers_cnt = 0
self.putWorkersInOrder()
for w_thread in self.workers:
w_thread.join()
self.results_queue.put('finish')
if __name__ == '__main__':
spider = Spider(common.getConfig())
try: spider.run()
except Exception as exception:
spider.uids.flushToDisk()
raise exception
