def getDBConn():
dbConn = {}
config = getConfig(["/etc/dtnrm-site-fe.conf"])
for sitename in config.get('general', 'sites').split(','):
if config.has_option(sitename, "database"):
dbConn[sitename] = dbinterface(config.get(sitename, "database"))
return dbConn
def getDBConn(serviceName=''):
"""Get datanase connection."""
dbConn = {}
config = getConfig()
for sitename in config.get('general', 'sites').split(','):
dbConn[sitename] = dbinterface(serviceName, config, sitename)
return dbConn
def check_initialized(environ):
"""Env and configuration initialization"""
global _INITIALIZED
global _CP
if not _INITIALIZED:
_CP = getConfig(["/etc/dtnrm-site-fe.conf"])
_INITIALIZED = True
def __init__(self, logger, config=None):
self.dbI = getDBConn()
self.config = getConfig(["/etc/dtnrm-site-fe.conf"])
if config:
self.config = config
self.logger = logger
self.policer = polS.PolicyService(self.config, self.logger)
self.stateM = stateM.StateMachine(self.logger)
self.siteDB = contentDB(logger=self.logger, config=self.config)
def execute(config=None, logger=None):
"""Main Execute"""
if not config:
config = getConfig()
if not logger:
component = 'LookUpService'
logger = getLogger("%s/%s/" % (config.get('general', 'logDir'), component), config.get(component, 'logLevel'))
for siteName in config.get('general', 'sites').split(','):
policer = LookUpService(config, logger, siteName)
policer.startwork()
def check_initialized(environ):
"""Env and configuration initialization"""
global _INITIALIZED
global _CP
global _SITES
if not _INITIALIZED:
_CP = getConfig()
for sitename in _CP.get('general', 'sites').split(','):
_SITES.append(sitename)
_INITIALIZED = True
def __init__(self, config, logger):
self.config = config if config else getConfig()
self.logger = logger if logger else getLogger(
"%s/%s/" % (self.config.get('general', 'logDir'), COMPONENT),
self.config.get('general', 'logLevel'))
self.fullURL = getFullUrl(self.config,
self.config.get('general', 'siteName'))
self.hostname = self.config.get('agent', 'hostname')
self.logger.info("====== Debugger Start Work. Hostname: %s",
self.hostname)
def execute(config=None, logger=None, args=None):
"""Main Execute"""
if not config:
config = getConfig(["/etc/dtnrm-site-fe.conf"])
if not logger:
component = 'LookUpService'
logger = getLogger(
"%s/%s/" % (config.get('general', 'logDir'), component),
config.get(component, 'logLevel'))
provisioner = ProvisioningService(config, logger, args)
provisioner.startwork()
def __init__(self, logger, config=None):
self.dbI = getDBConn('REST-DELTA')
self.config = getConfig()
if config:
self.config = config
self.logger = logger
self.policer = {}
for sitename in self.config.get('general', 'sites').split(','):
policer = polS.PolicyService(config, logger, sitename)
self.policer[sitename] = policer
self.stateM = stateM.StateMachine(self.logger)
self.siteDB = contentDB(logger=self.logger, config=self.config)
def __init__(self, config, logger):
self.config = config if config else getConfig()
self.logger = logger if logger else getLogger(
"%s/%s/" % (self.config.get('general', 'logDir'), COMPONENT),
self.config.get('general', 'logLevel'))
self.workDir = self.config.get('general',
'private_dir') + "/DTNRM/QOS/"
self.configDir = self.config.get('general',
'private_dir') + "/DTNRM/RulerAgent/"
self.hostname = self.config.get('agent', 'hostname')
createDirs(self.workDir)
self.debug = self.config.getboolean('general', "debug")
self.agentdb = contentDB(logger=self.logger, config=self.config)
def getNetdataConfig():
"""
Get netdata server config and also check backend configuration
Returns: boolean: True - success in getting config and all parameters defined correctly.
False - Failure in retrieving one or another configuration parameters.
dict: {} - if boolean return is False, always empty as failed to retrieve.
otherwise dictionary of all parsed configuration parameters
"""
outDict = {}
with tempfile.NamedTemporaryFile(delete=False) as fd:
try:
tmpConf = makeGETRequest("http://localhost:19999/netdata.conf")
for line in tmpConf[1].splitlines():
fd.write(line.replace('\t', '') + "\n")
except urllib2.URLError as ex:
print 'Received URLError %s. Checking if config file is present locally' % ex
return False, {}
tempfileName = fd.name
hostConfig = getConfig([tempfileName])
if not hostConfig.has_section('backend'):
print 'Netdata server is not configured to publish anything to any backend'
print '* Skipping this node check.'
return False, {}
if not hostConfig.has_option('backend', 'enabled'):
print 'Netdata does not have enabled option in backend configuration'
print '* Skipping this node check.'
return False, {}
if not hostConfig.has_option('backend', 'destination'):
print 'Netdata is configured to send metrics but destination is not set.'
print '* Skipping this node check.'
return False, {}
for optionKey in hostConfig.options('backend'):
outDict[optionKey] = hostConfig.get('backend', optionKey)
# Make boolean from send names instead of ids
if 'send names instead of ids' in outDict:
if outDict['send names instead of ids'] == 'yes':
outDict['dimensionids'] = True
elif outDict['send names instead of ids'] == 'no':
outDict['dimensionids'] = False
else:
outDict['dimensionids'] = True
else:
outDict['dimensionids'] = False
outDict['send names instead of ids'] = 'no'
if 'host tags' in outDict:
outDict['tags'] = {}
for item in outDict['host tags'].split(" "):
tmpTags = item.split('=')
outDict['tags'][tmpTags[0]] = tmpTags[1]
# host tags = hostname=sensei3.ultralight.org nodetype=sense-service sitename=T2_US_Caltech
return True, outDict
def __init__(self, config, logger):
self.config = config if config else getConfig()
self.logger = logger if logger else getLogger(
"%s/%s/" % (self.config.get('general', 'logDir'), COMPONENT),
self.config.get('general', 'logLevel'))
self.workDir = self.config.get('general',
'private_dir') + "/DTNRM/RulerAgent/"
createDirs(self.workDir)
self.fullURL = getFullUrl(self.config,
self.config.get('general', 'siteName'))
self.noRules = self.config.getboolean('agent', 'norules')
self.hostname = self.config.get('agent', 'hostname')
self.logger.info("====== Ruler Start Work. Hostname: %s",
self.hostname)
self.vInterface = VInterfaces(self.config, self.logger)
def execute(config=None, logger=None, args=None):
"""Main Execute"""
if not config:
config = getConfig(["/etc/dtnrm-site-fe.conf"])
if not logger:
component = 'PolicyService'
logger = getLogger(
"%s/%s/" % (config.get('general', 'logDir'), component),
config.get(component, 'logLevel'), True)
policer = PolicyService(config, logger)
if args:
print policer.parseDeltaRequest(args[1], [], args[2])
else:
policer.startwork()
def execute(config=None, logger=None, args=None):
"""Main Execute."""
if not config:
config = getConfig()
if not logger:
component = 'LookUpService'
logger = getLogger(
"%s/%s/" % (config.get('general', 'logDir'), component),
config.get(component, 'logLevel'))
if args:
provisioner = ProvisioningService(config, logger, args[1])
provisioner.startwork()
else:
for sitename in config.get('general', 'sites').split(','):
provisioner = ProvisioningService(config, logger, sitename)
provisioner.startwork()
def execute(config=None, logger=None, args=None):
"""Main Execute"""
if not config:
config = getConfig()
if not logger:
component = 'PolicyService'
logger = getLogger("%s/%s/" % (config.get('general', 'logDir'), component),
config.get(component, 'logLevel'), True)
if args:
policer = PolicyService(config, logger, args[3])
# This is only for debugging purposes.
print policer.parseDeltaRequest(args[1], {args[2]: []})
else:
for sitename in config.get('general', 'sites').split(','):
policer = PolicyService(config, logger, sitename)
policer.startwork()
outputForFE["interfaces"].setdefault(
mainInf, {'vlans': {}})
outputForFE["interfaces"][mainInf]['vlans'][
vlankey] = vlandict
else:
print 'This interface was defined in configuration, but not available. Will not add it to final output'
print intfName, intfDict
# Get Routing Information
outputForFE["routes"] = getRoutes(config)
return outputForFE
def getRoutes(config):
""" Get Routing information from host """
del config
routes = []
with IPRoute() as ipr:
for route in ipr.get_routes(table=254, family=2):
newroute = {"dst_len": route['dst_len']}
for item in route['attrs']:
if item[0] in ['RTA_GATEWAY', 'RTA_DST', 'RTA_PREFSRC']:
newroute[item[0]] = item[1]
routes.append(newroute)
print routes
return routes
if __name__ == "__main__":
PRETTY = pprint.PrettyPrinter(indent=4)
PRETTY.pprint(get(getConfig(), getStreamLogger()))
#!/usr/bin/env python
""" List all deltas inFrontend """
import sys
from DTNRMLibs.MainUtilities import getVal
from DTNRMLibs.MainUtilities import evaldict
from DTNRMLibs.MainUtilities import getConfig, getStreamLogger
from DTNRMLibs.FECalls import getDBConn
from SiteFE.PolicyService.stateMachine import StateMachine
LOGGER = getStreamLogger()
CONFIG = getConfig()
STATEMACHINE = StateMachine(LOGGER)
def getdeltaAll(sitename):
dbI = getDBConn('listalldeltas')
dbobj = getVal(dbI, sitename=sitename)
for delta in dbobj.get('deltas'):
delta['addition'] = evaldict(delta['addition'])
delta['reduction'] = evaldict(delta['reduction'])
print '='*80
print 'Delta UID : ', delta['uid']
print 'Delta RedID: ', delta['reductionid']
print 'Delta State: ', delta['state']
print 'Delta ModAdd: ', delta['modadd']
print 'Delta InsDate:', delta['insertdate']
print 'Delta Update: ', delta['updatedate']
print 'Delta Model: ', delta['modelid']
print 'Delta connID: ', delta['connectionid']
print 'Delta Deltatype: ', delta['deltat']
from DTNRMAgent.RecurringActions.Utilities import externalCommand, tryConvertToNumeric
from DTNRMLibs.MainUtilities import getConfig
NAME = 'MemInfo'
def get(config):
"""Get memory info from /proc/meminfo"""
del config
memInfo = {}
tmpOut = externalCommand('cat /proc/meminfo')
for item in tmpOut:
for desc in item.split('\n'):
vals = desc.split(':')
if len(vals) == 2:
value = vals[1].strip().split(' ')
# We strip it to remove white spaces and split to remove kb in the end
name = vals[0].strip()
if len(value) == 2:
name += "_%s" % value[1]
memInfo[name] = tryConvertToNumeric(value[0])
else:
print 'MemInfo: Skipped this item: ', vals
memInfo['memory_mb'] = int(memInfo['MemTotal_kB'] / 1000)
return memInfo
if __name__ == "__main__":
PRETTY = pprint.PrettyPrinter(indent=4)
PRETTY.pprint(get(getConfig(['/etc/dtnrm/main.conf', 'main.conf'])))
def __init__(self):
self.dbI = getDBConn('REST-Frontend')
self.initialized = False
self.config = getConfig()
self.siteDB = contentDB()
Email : justas.balcas (at) cern.ch
@Copyright : Copyright (C) 2016 California Institute of Technology
Date : 2017/09/26
"""
from shutil import copy2
from DTNRMLibs.MainUtilities import evaldict
from DTNRMLibs.MainUtilities import readFile
from DTNRMLibs.MainUtilities import getLogger
from DTNRMLibs.MainUtilities import getConfig
from DTNRMLibs.MainUtilities import getFullUrl
from DTNRMLibs.MainUtilities import createDirs
from DTNRMLibs.MainUtilities import externalCommand
from DTNRMLibs.MainUtilities import getDataFromSiteFE
from DTNRMLibs.CustomExceptions import FailedToParseError
CONFIG = getConfig(["/etc/dtnrm-site-fe.conf"])
COMPONENT = 'ForwardingService'
LOGGER = getLogger("%s/%s/" % (CONFIG.get('general', 'logDir'), COMPONENT),
CONFIG.get(COMPONENT, 'logLevel'))
def checkConsistency(inputDict, currentHttpdConf, start, end):
"""Check current config and information from Site FE. Append what is missing."""
newConfig = []
changed = False
print 'Appending the beginning %s lines' % start
for lineNum in range(0, start):
newConfig.append(currentHttpdConf[lineNum])
print 'Starting to check current configuration with what we received from DB'
newConfig.append("### HERE STARTS PROXYREWRITERULES")
for lineNum in range(start, end):
def __init__(self):
self.dbI = getDBConn()
self.initialized = False
self.config = getConfig(["/etc/dtnrm-site-fe.conf"])
self.siteDB = contentDB()
