def getNetdataConfig(mainConfig, sectionName): """ Get netdata server config and also check backend configuration Input: Config: mainConfig : All configuration str sectionName : Section Name of that specific config Returns: boolean: True - success in getting config and all parameters defined correctly. False - Failure in retrieving one or another configuration parameters. dict: {} - if boolean return is False, always empty as failed to retrieve. otherwise dictionary of all parsed configuration parameters """ # TODO look at netdata filters, and ignore which are not published to Database hostname = mainConfig.get(sectionName, 'hostname') outDict = {} with tempfile.NamedTemporaryFile(delete=False) as fd: try: tmpConf = getDataFromUrl("%s/netdata.conf" % hostname) for line in tmpConf.read().splitlines(): fd.write(line.replace('\t', '') + "\n") except urllib2.URLError as ex: print 'Received URLError %s for %s. Checking if config file is present locally' % ( ex, hostname) if os.path.isfile("%s.conf" % sectionName): with open("%s.conf" % sectionName, 'r') as fdConf: for line in fdConf: fd.write(line.replace('\t', '') + "\n") else: print 'Config file is also not present. Skipping this node %s check' % hostname return False, {} tempfileName = fd.name hostConfig = getConfig([tempfileName]) if not hostConfig.has_section('backend'): print 'Hostname %s Netdata server is not configured to publish anything to any backend' % hostname print '* Skipping this node check.' return False, {} if not hostConfig.has_option('backend', 'enabled'): print 'Hostname %s Netdata does not have enabled option in backend configuration' % hostname print '* Skipping this node check.' return False, {} if not hostConfig.has_option('backend', 'destination'): print 'Hostname %s Netdata is configured to send metrics but destination is not set.' % hostname print '* Skipping this node check.' return False, {} if not hostConfig.has_option('backend', 'prefix'): outDict['prefix'] = 'netdata' for optionKey in hostConfig.options('backend'): outDict[optionKey] = hostConfig.get('backend', optionKey) print outDict # Make boolean from send names instead of ids if 'send names instead of ids' in outDict: if outDict['send names instead of ids'] == 'yes': outDict['dimensionids'] = True elif outDict['send names instead of ids'] == 'no': outDict['dimensionids'] = False else: outDict['dimensionids'] = True else: outDict['dimensionids'] = False outDict['send names instead of ids'] = 'no' return True, outDict
def get_cron_task(self): if self.pkg_conf is None: self.pkg_conf = get_pkg_config(self.package_id, self.package_path) self.pkg_cron = common.getConfig(self.pkg_conf, 'crontab') if self.pkg_cron == "": return 0 return 0
def __init__(self): super(test_ALI_5, self).__init__("test_ALI_5") common.sensemediaTestBase.setlogger(self, __name__) self.testid="ALI-5" #下文的self.tpid self.testplanid="" #建议获取最新的buildname(),下文中的self.build_name self.buildname="" #eg:"2019-01-18 14:55" self.timestamp=time.strftime("%Y-%m-%d %H:%m") self.testcasestarttime=time.time() self.execduration="" #获取TestLink 该用例的一些信息 self.testlink_url = common.getConfig("url", "testlinkurl") self.testlink_key = common.getConfig("url", "testlinkkey") self.project_id = common.getConfig("url", "project_id") self.project_name = common.getConfig("url", "project_name") self.logger.info("project name is %s "% self.project_name) self.testPlanName = common.getConfig("url", "testPlanName")
def getDBlocation(retvar): custdb = addon.getSetting('customdbfolder') == 'true' old_dbpath = xbmc.translatePath(getConfig('old_dbfolder')).decode('utf-8') cur_dbpath = dbpath if not old_dbpath: old_dbpath = cur_dbpath if custdb: cur_dbpath = xbmc.translatePath( addon.getSetting('dbfolder')).decode('utf-8') else: addon.setSetting('dbfolder', dbpath) orgDBfile = { 'tv': os.path.join(dbpath, 'tv.db'), 'movie': os.path.join(dbpath, 'movies.db') } oldDBfile = { 'tv': os.path.join(old_dbpath, 'tv.db'), 'movie': os.path.join(old_dbpath, 'movies.db') } DBfile = { 'tv': os.path.join(cur_dbpath, 'tv.db'), 'movie': os.path.join(cur_dbpath, 'movies.db') } if old_dbpath != cur_dbpath: Log('DBPath changed') if xbmcvfs.exists(oldDBfile['tv']) and xbmcvfs.exists( oldDBfile['movie']): if not xbmcvfs.exists(cur_dbpath): xbmcvfs.mkdir(cur_dbpath) if not xbmcvfs.exists(DBfile['tv']) or not xbmcvfs.exists( DBfile['movie']): copyDB(oldDBfile, DBfile) writeConfig('old_dbfolder', cur_dbpath) if custdb: org_fileacc = int( xbmcvfs.Stat(orgDBfile['tv']).st_mtime() + xbmcvfs.Stat(orgDBfile['movie']).st_mtime()) cur_fileacc = int( xbmcvfs.Stat(DBfile['tv']).st_mtime() + xbmcvfs.Stat(DBfile['movie']).st_mtime()) if org_fileacc > cur_fileacc: copyDB(orgDBfile, DBfile, True) return DBfile[retvar]
def get_config(self, config_file): clear_conf = common.getConfig(config_file, 'clear_file') # 处理以#开头的行 regex = r"\s*#" conf_arr = clear_conf.splitlines() real_conf = [] for line in conf_arr: ret = re.match(regex, line) if ret is None: # 处理#注释在行尾的情况 reg_2 = r"^((\"[^\"]*\"|'[^']*'|[^'\"#])*)(#.*)$" ret = re.match(reg_2, line) if ret is not None: conf_line = ret.group(1) else: conf_line = line conf = re.split(r'\s+', conf_line) if len(conf) < 5: continue real_conf.append(conf) return real_conf
def main(): parser = OptionParser() parser.add_option("-b", "--bot", dest="bot", default="me", help="IDREF of bot in config file") parser.add_option("-p", "--pubsub", dest="pubsub", default="tune", help="IDREF of pubsub in config file") (options, args) = parser.parse_args() config = common.getConfig() try: bot_jid=xmpp.protocol.JID(config("users/user[@id='%s']/jid/text()"%(options.bot))[0].nodeValue + '/tunepub/mpd') bot_passwd = config("users/user[@id='%s']/password/text()"%(options.bot))[0].nodeValue pubsub_service = config("pubsub/node[@id='%s']/service/text()"%(options.pubsub))[0].nodeValue pubsub_id = config("pubsub/node[@id='%s']/id/text()"%(options.pubsub))[0].nodeValue except: print 'Could not find proper nodes in config file (Looked for bot \'%s\' and pubsub node \'%s\')'%(options.bot, options.pubsub) sys.exit() c = None last_song = '' client = xmpp.Client(bot_jid.getDomain(), debug=[]) client.connect() client.auth(bot_jid.getNode(), bot_passwd) while True: if c == None: try: c = mpdclient.MpdController(host="localhost", port=6600) except: c = None if c == None: mpd_song = False else: try: mpd_song = c.getCurrentSong() except: c = None mpd_song = False try: if mpd_song == False: raise Exception status = c.getStatus() title = mpd_song.title length = status.totalTime artist = mpd_song.artist source = mpd_song.path if artist == '' and title.count(' - ') > 0: parts = title.split(' - ') artist = parts[0] title = parts[1] if last_song != title: escape = xmpp.simplexml.XMLescape node_id = md5.md5(time.time().__str__()).hexdigest() node_xml = "<iq type='set' from='%s' to='%s' id='%s'><pubsub xmlns='http://jabber.org/protocol/pubsub'><publish node='%s'><item id='current'><tune xmlns='http://jabber.org/protocol/tune'><artist>%s</artist><title>%s</title><source>%s</source><length>%s</length></tune></item></publish></pubsub></iq>"%(bot_jid,pubsub_service,node_id,pubsub_id,escape(artist),escape(title),escape(source),length) tune_node = xmpp.simplexml.XML2Node(node_xml) tune_iq = xmpp.protocol.Iq(node=tune_node) except: title = 'Stopped' if last_song != title: node_id = md5.md5(time.time().__str__()).hexdigest() node_xml = "<iq type='set' from='%s' to='%s' id='%s'><pubsub xmlns='http://jabber.org/protocol/pubsub'><publish node='%s'><item id='current'><tune xmlns='http://jabber.org/protocol/tune' /></item></publish></pubsub></iq>"%(bot_jid,pubsub_service,node_id,pubsub_id) tune_node = xmpp.simplexml.XML2Node(node_xml) tune_iq = xmpp.protocol.Iq(node=tune_node) if last_song != title: client.send(tune_iq) last_song = title try: time.sleep(5) except KeyboardInterrupt: sys.exit()
def main(): """ Main execution. For all config specified netdata hosts, prepare grafana dashboard template. Input: None Returns: None """ # TODO. Allow to specify configuration location. allConfigs = {"HOSTS": {}} mainConfig = getConfig(['netdata-grafana-hosts.conf']) allConfigs['backend'] = mainConfig.get('global', 'backend') allConfigs['grafanaUrl'] = mainConfig.get('global', 'grafanaUrl') if allConfigs['backend'] == 'opentsdb': allConfigs['opentsdb'] = { "datasource": mainConfig.get('opentsdb', 'datasource'), "order": checkOrderConfig(mainConfig, 'opentsdb'), "skipOthers": checkSkipOthers(mainConfig, 'opentsdb') } allConfigs['opentsdb']['title'] = mainConfig.get('opentsdb', 'title') allConfigs['opentsdb']['description'] = mainConfig.get( 'opentsdb', 'description') allConfigs['opentsdb']['dimensionids'] = mainConfig.getboolean( 'opentsdb', 'dimensionids') allConfigs['opentsdb']['prefix'] = mainConfig.get('opentsdb', 'prefix') allConfigs['opentsdb']['tags'] = getTags(mainConfig, 'opentsdb') allConfigs['opentsdb']['customfilters'] = json.loads( mainConfig.get('opentsdb', 'customfilters')) # get customFirstRow and customLastRow allConfigs['opentsdb']['customFirstRow'] = getValFromConfig( mainConfig, 'opentsdb', 'customFirstRow') allConfigs['opentsdb']['customLastRow'] = getValFromConfig( mainConfig, 'opentsdb', 'customLastRow') for sectionName in mainConfig.sections(): if sectionName in ['global', 'opentsdb']: continue # check if mandatory options are in place if not (mainConfig.has_option(sectionName, 'hostname') and mainConfig.get(sectionName, 'hostname')): print 'In section %s hostname is not defined. It is mandatory to define full url' % sectionName print '* Skipping this node check.' continue if allConfigs['backend'] == 'graphite': if not (mainConfig.has_option(sectionName, 'datasource') and mainConfig.get(sectionName, 'datasource')): print 'In section %s dataspirce is not defined. It is mandatory to define datasource' % sectionName print '* Skipping this node check.' continue configSuccess, config = getNetdataConfig(mainConfig, sectionName) if not configSuccess: config[ 'SKIP_NODE'] = False # This is not looked in case of graphite. TODO config['tags'] = getTags(mainConfig, allConfigs['backend']) if allConfigs['backend'] == 'graphite': # This is relevant only for graphite config['datasource'] = mainConfig.get(sectionName, 'datasource') config['order'] = checkOrderConfig(mainConfig, sectionName) config['skipOthers'] = checkSkipOthers(mainConfig, sectionName) config['hostname'] = mainConfig.get(sectionName, 'hostname') config['section'] = sectionName # get customFirstRow and customLastRow config['customFirstRow'] = getValFromConfig(mainConfig, sectionName, 'customFirstRow') config['customLastRow'] = getValFromConfig(mainConfig, sectionName, 'customLastRow') allConfigs["HOSTS"][config['hostname']] = config print allConfigs # Now send allConfigs to a specific backend preparator. if allConfigs['backend'] == 'graphite': graphiteDashboard(allConfigs) elif allConfigs['backend'] == 'opentsdb': opentsdbDashboard(allConfigs) else: print 'Unknown backend type... Exiting'
#!/usr/bin/python import sys,os,xmpp,common from time import time from optparse import OptionParser from pysqlite2 import dbapi2 as sqlite parser = OptionParser() parser.add_option("-b", "--bot", dest="bot", default="bot", help="IDREF of bot in config file") (options, args) = parser.parse_args() config = common.getConfig() try: bot_jid=xmpp.protocol.JID(config("//user[@id='%s']/jid/text()"%(options.bot))[0].nodeValue + '/presencebot') bot_passwd = config("//user[@id='%s']/password/text()"%(options.bot))[0].nodeValue except: print 'Could not find proper nodes in config file (Looked for bot \'%s\')'%(options.bot) sys.exit(); presence_db = sqlite.connect(config("//db/text()")[0].nodeValue,isolation_level=None) tune_db = sqlite.connect(config("//db/text()")[0].nodeValue,isolation_level=None) def main(): cur = presence_db.cursor() # cur.execute('DELETE FROM presence') client = xmpp.Client(bot_jid.getDomain(), debug=[]) client.connect() client.auth(bot_jid.getNode(), bot_passwd, bot_jid.getResource()) client.sendInitPresence() client.RegisterHandler('presence', do_presence) client.RegisterHandler('message', do_message) while step(client):
def reportStatus(self, pkg_id, pkg_conf_path, inst_info, status, err_info={}): reportInfo = {} reportInfo['dims'] = {} reportInfo['vals'] = {} reportInfo['dims']['process.process.package_id'] = pkg_id reportInfo['dims']['process.process.version_id'] = "" reportInfo['dims']['process.process.install_path'] = inst_info[ 'installPath'] reportInfo['vals']['process.process.package_status'] = status if not err_info.get('err_proc') and not err_info.get('err_port'): reportInfo['vals']['process.process.alert_status'] = 0 else: reportInfo['vals']['process.process.alert_status'] = 1 conf_file = os.path.join(pkg_conf_path, 'package.conf.yaml') proc_config = common.getConfig(conf_file, "proc_list") port_config = common.getConfig(conf_file, "port_list") proc_list = {} for proc in proc_config: proc_list[proc['proc_name']] = proc normal_proc_list = [] proc_num_list = self.pkgCom.getProcNum() normal_proc_str = "" for proc_name in err_info.get('ok_proc', []): num_min = proc_list[proc_name].get('proc_num_min', 0) num_max = proc_list[proc_name].get('proc_num_max', 0) proc_str = "%s:%s,%s|%s" % (proc_name, num_min, num_max, proc_num_list[proc_name]) normal_proc_list.append(proc_str) normal_proc_str = '##'.join(normal_proc_list) abnormal_proc_list = [] abnormal_proc_str = "" for proc_name in err_info.get('err_proc', []): num_min = proc_list[proc_name].get('proc_num_min', 0) num_max = proc_list[proc_name].get('proc_num_max', 0) proc_str = "%s:%s,%s|%s" % (proc_name, num_min, num_max, proc_num_list[proc_name]) abnormal_proc_list.append(proc_str) abnormal_proc_str = '##'.join(abnormal_proc_list) normal_port_list = [] normal_port_str = "" for port in err_info.get('ok_port', []): normal_port_list.append(port) normal_port_str = "##".join(map(str, normal_port_list)) abnormal_port_list = [] abnormal_port_str = "" for port in err_info.get('err_port', []): abnormal_port_list.append(port) abnormal_port_str = "##".join(map(str, abnormal_port_list)) reportInfo['vals'][ 'process.process.normal_processes'] = normal_proc_str reportInfo['vals'][ 'process.process.abnormal_processes'] = abnormal_proc_str reportInfo['vals']['process.process.normal_ports'] = normal_port_str reportInfo['vals'][ 'process.process.abnormal_ports'] = abnormal_port_str if report: ret, msg = report(data_id=3000, dims=reportInfo['dims'], vals=reportInfo['vals'])
def monitorPkg(self, pkg_info): import random gevent.sleep(random.randint(0, 15)) pkg_id, pkg_conf_path, pkg_install_path = pkg_info[ 'packageId'], pkg_info['confPath'], pkg_info['installPath'] # 初始化log配置 st = time.time() mLog = self.pkgCom.configLog(pkg_install_path, 'monitor', self.log_level) mLog.info('check process start: %s' % pkg_install_path) # 读取实例信息 inst_conf_file = os.path.join(pkg_conf_path, 'instance.conf.yaml') with open(inst_conf_file, 'r') as fp: conf_info = yaml.load(fp) inst_info = conf_info.pop() # 检查包是否已启动 status_conf_file = os.path.join(pkg_conf_path, 'package.status') if not os.path.isfile(status_conf_file): return 0, 'ok' with open(status_conf_file, 'r') as fp: status_info = yaml.load(fp) start_status = status_info['status'] if start_status == 'stopped': self.reportStatus(pkg_id, pkg_conf_path, inst_info, status=start_status) mLog.info('package status is stopped: %s %s' % (pkg_info['installPath'], round(time.time() - st, 4))) return 0, 'ok' # 获得文件锁,准备检查。且快速失败,因为进程检查本身就是1分钟执行一次的 ret = self.pkgCom.getLock(pkg_conf_path, timeout=10) if not ret: mLog.error("get lock error") return 2008, "get lock error" # 根据包配置信息检查包的进程状态 conf_file = os.path.join(pkg_conf_path, 'package.conf.yaml') err_proc, ok_proc = self.pkgCom.checkProcStatus( conf_file, install_path=inst_info.get('installPath')) err_port, ok_port = self.pkgCom.checkPort( conf_file, install_path=inst_info.get('installPath')) proc_config = common.getConfig(conf_file, "proc_guard") port_config = common.getConfig(conf_file, "port_guard") err_info = { 'err_proc': err_proc, 'ok_proc': ok_proc, 'err_port': err_port, 'ok_port': ok_port } self.reportStatus(pkg_id, pkg_conf_path, inst_info, status=start_status, err_info=err_info) code = 0 msg = 'ok' err_msg = "" if err_proc: err_msg += ",error process:" + ",".join(err_proc) if err_port: err_msg += ",error port:" + ",".join(map(str, err_port)) # 包操作对象 op = pkgOp(self.APP_BASE, inst_info['installPath']) if (err_proc and proc_config == 'stopStart') or (err_port and port_config == 'stopStart'): msg = "process error,monitor run stopStart:" + err_msg mLog.info(msg) code, msg = op.stopStart(inst_info['packageId'], inst_info['installPath']) elif (err_proc and proc_config == 'custom') or (err_port and port_config == 'custom'): msg = "process error,monitor run custom script:" + err_msg mLog.info(msg) code, msg = op.resolve(inst_info['packageId'], inst_info['installPath']) elif err_proc or err_port: msg = "process error,do nothing:" + err_msg mLog.info(msg) # 解锁 self.pkgCom.unLock(pkg_conf_path) mLog.info('check process end: %s %s' % (pkg_info['installPath'], round(time.time() - st, 4))) return code, msg
self.log.rotateLogs() #ротируем директории рабочих worker.createEmptyDirs(*self.worker_dirs) #запускаем рабочих self.workers_cnt = workers_cnt self.putWorkersInOrder() def handleSIGTERM(self, sig_num, frame): self.log.write('OK. FINISHING') #очищаю список задач with open(self.tasks_file, 'w') as output: while not self.tasks_queue.empty(): output.write(self.tasks_queue.get() + '\n') #убиваем всех рабочих workers_cnt = self.workers_cnt self.workers_cnt = 0 self.putWorkersInOrder() for w_thread in self.workers: w_thread.join() self.results_queue.put('finish') if __name__ == '__main__': spider = Spider(common.getConfig()) try: spider.run() except Exception as exception: spider.uids.flushToDisk() raise exception