def __init__(self, config, logger, sitename):
self.sitename = sitename
self.logger = logger
self.config = config
self.siteDB = contentDB(logger=self.logger, config=self.config)
self.dbI = getDBConn('PolicyService')
self.stateMachine = StateMachine(self.logger)
#!/usr/bin/env python
""" List all deltas inFrontend """
import sys
from DTNRMLibs.MainUtilities import getVal
from DTNRMLibs.MainUtilities import evaldict
from DTNRMLibs.MainUtilities import getConfig, getStreamLogger
from DTNRMLibs.FECalls import getDBConn
from SiteFE.PolicyService.stateMachine import StateMachine
LOGGER = getStreamLogger()
CONFIG = getConfig()
STATEMACHINE = StateMachine(LOGGER)
def getdeltaAll(sitename):
dbI = getDBConn('listalldeltas')
dbobj = getVal(dbI, sitename=sitename)
for delta in dbobj.get('deltas'):
delta['addition'] = evaldict(delta['addition'])
delta['reduction'] = evaldict(delta['reduction'])
print '='*80
print 'Delta UID : ', delta['uid']
print 'Delta RedID: ', delta['reductionid']
print 'Delta State: ', delta['state']
print 'Delta ModAdd: ', delta['modadd']
print 'Delta InsDate:', delta['insertdate']
print 'Delta Update: ', delta['updatedate']
print 'Delta Model: ', delta['modelid']
print 'Delta connID: ', delta['connectionid']
print 'Delta Deltatype: ', delta['deltat']
def __init__(self, config, logger):
self.logger = logger
self.config = config
self.siteDB = contentDB(logger=self.logger, config=self.config)
self.dbI = getDBConn()
self.stateMachine = StateMachine(self.logger)
class PolicyService(object):
""" Policy Service to accept deltas """
def __init__(self, config, logger, sitename):
self.sitename = sitename
self.logger = logger
self.config = config
self.siteDB = contentDB(logger=self.logger, config=self.config)
self.dbI = getDBConn('PolicyService')
self.stateMachine = StateMachine(self.logger)
def queryGraph(self, graphIn, sub=None, pre=None, obj=None, search=None):
""" Does search inside the graph based on provided parameters """
foundItems = []
self.logger.debug('Searching for subject: %s predica: %s object: %s searchLine: %s' % (sub, pre, obj, search))
for sIn, pIn, oIn in graphIn.triples((sub, pre, obj)):
if search:
if search == pIn:
self.logger.debug('Found item with search parameter')
self.logger.debug("s(subject) %s" % sIn)
self.logger.debug("p(predica) %s" % pIn)
self.logger.debug("o(object ) %s" % oIn)
self.logger.debug("-" * 50)
foundItems.append(oIn)
else:
self.logger.debug('Found item without search parameter')
self.logger.debug("s(subject) %s" % sIn)
self.logger.debug("p(predica) %s" % pIn)
self.logger.debug("o(object ) %s" % oIn)
self.logger.debug("-" * 50)
foundItems.append(oIn)
return foundItems
def getTimeScheduling(self, out, gIn, prefixes):
""" This is for identifying LIFETIME! In case it fails to get correct timestamp,
resources will be provisioned right away
"""
for timeline in out:
times = {}
for timev in ['end', 'start']:
tout = self.queryGraph(gIn, timeline, search=URIRef('%s%s' % (prefixes['nml'], timev)))
temptime = None
try:
temptime = int(time.mktime(parser.parse(str(tout[0])).timetuple()))
if time.daylight:
temptime -= 3600
except Exception:
continue
times[timev] = temptime
if len(times.keys()) == 2:
return times
return {}
def parseDeltaRequest(self, inFileName, allKnownHosts):
"""Parse delta request to json"""
self.logger.info("Parsing delta request %s ", inFileName)
prefixes = {}
allOutput = []
prefixes['site'] = "%s:%s:%s" % (self.config.get('prefixes', 'site'),
self.config.get(self.sitename, 'domain'),
self.config.get(self.sitename, 'year'))
for switchName in self.config.get(self.sitename, 'switch').split(','):
try:
vsw = self.config.get(switchName, 'vsw')
except ConfigParser.NoOptionError:
self.logger.debug('ERROR: vsw parameter is not defined for %s.', switchName)
continue
prefixes['main'] = URIRef("%s:service+vsw:%s" % (prefixes['site'], vsw))
prefixes['nml'] = self.config.get('prefixes', 'nml')
prefixes['mrs'] = self.config.get('prefixes', 'mrs')
gIn = Graph()
gIn.parse(inFileName, format='turtle')
out = []
self.logger.info('Trying to parse L2 info from delta')
out = self.parsel2Request(allKnownHosts, prefixes, gIn, out)
self.logger.info('Trying to parse L3 info from delta')
out = self.parsel3Request(allKnownHosts, prefixes, gIn, out)
allOutput.append(out)
allOutput = list(filter(None, allOutput))
if len(allOutput) > 1:
msg = 'Got multiple Service definitions. Not Supported. Output: %s' % allOutput
self.logger.info(msg)
return {"errorType": 'MultipleDefs',
"errorNo": '-9',
"errMsg": msg}
elif not allOutput:
return []
return allOutput[0]
def parsel3Request(self, allKnownHosts, prefixes, gIn, returnout):
""" Parse Layer 3 Delta Request """
for hostname in allKnownHosts.keys():
prefixes['mainrst'] = URIRef("%s:%s:service+rst" % (prefixes['site'], hostname))
self.logger.info('Lets try to get connection ID subject for %s' % prefixes['mainrst'])
out = self.queryGraph(gIn, prefixes['mainrst'],
search=URIRef('%s%s' % (prefixes['mrs'], 'providesRoutingTable')))
if not out:
msg = 'Connection ID was not received. Continue'
self.logger.info(msg)
continue
outall = {}
outall.setdefault('hosts', {})
outall['hosts'].setdefault(hostname, {})
for connectionID in out:
outall['connectionID'] = str(connectionID)
outall['hosts'][hostname]['routes'] = []
self.logger.info('This is our connection ID: %s' % connectionID)
self.logger.info('Now lets get all info what it wants to do. Mainly nextHop, routeFrom, routeTo')
bidPorts = self.queryGraph(gIn, connectionID, search=URIRef('%s%s' % (prefixes['mrs'], 'hasRoute')))
for bidPort in bidPorts:
route = {}
for flag in ['nextHop', 'routeFrom', 'routeTo']:
route.setdefault(flag, {})
out = self.queryGraph(gIn, bidPort, search=URIRef('%s%s' % (prefixes['mrs'], flag)))
if not out:
continue
for item in out:
outt = self.queryGraph(gIn, item, search=URIRef('%s%s' % (prefixes['mrs'], 'type')))
outv = self.queryGraph(gIn, item, search=URIRef('%s%s' % (prefixes['mrs'], 'value')))
if not outt or not outv:
continue
route[flag]['type'] = str(outt[0])
route[flag]['value'] = str(outv[0])
outall['hosts'][hostname]['routes'].append(route)
returnout.append(outall)
self.logger.debug('L3 Parse output: %s', outall)
return returnout
def parsel2Request(self, allKnownHosts, prefixes, gIn, returnout):
""" Parse L2 request """
self.logger.info('Lets try to get connection ID subject for %s' % prefixes['main'])
connectionID = None
out = self.queryGraph(gIn, prefixes['main'], search=URIRef('%s%s' % (prefixes['mrs'], 'providesSubnet')))
if not out:
msg = 'Connection ID was not received. Something is wrong...'
self.logger.info(msg)
return []
for connectionID in out:
output = {}
output['connectionID'] = str(connectionID)
self.logger.info('This is our connection ID: %s' % connectionID)
self.logger.info('Now lets get all info what it wants to do. Mainly bidPorts and labelSwapping flag')
bidPorts = self.queryGraph(gIn, connectionID, search=URIRef('%s%s' % (prefixes['nml'],
'hasBidirectionalPort')))
out = self.queryGraph(gIn, connectionID, search=URIRef('%s%s' % (prefixes['nml'], 'labelSwapping')))
output['labelSwapping'] = str(out[0])
out = self.queryGraph(gIn, connectionID, search=URIRef('%s%s' % (prefixes['nml'], 'existsDuring')))
out = self.getTimeScheduling(out, gIn, prefixes)
if len(out.keys()) == 2:
output['timestart'] = out['start']
output['timeend'] = out['end']
# =======================================================
self.logger.info('Now lets get all info for each bidirectionalPort, like vlan, ip, serviceInfo ')
# We need mainly hasLabel, hasNetworkAddress
for bidPort in bidPorts:
print bidPort
# Get first which labels it has. # This provides us info about vlan tag
connInfo, output = getConnInfo(bidPort, prefixes['site'], output, nostore=True)
if connInfo not in allKnownHosts:
print 'Ignore %s' % connInfo
continue
connInfo, output = getConnInfo(bidPort, prefixes['site'], output)
alias = self.queryGraph(gIn, bidPort, search=URIRef('%s%s' % (prefixes['nml'], 'isAlias')))
if alias and alias[0] not in bidPorts:
self.logger.info('Received alias for %s to %s' % (bidPort, alias))
bidPorts.append(alias[0])
# Now let's get vlan ID
out = self.queryGraph(gIn, bidPort, search=URIRef('%s%s' % (prefixes['nml'], 'hasLabel')))
if not out:
continue
out = self.queryGraph(gIn, out[0], search=URIRef('%s%s' % (prefixes['nml'], 'value')))
output['hosts'][connInfo]['vlan'] = str(out[0])
# Now Let's get IP
out = self.queryGraph(gIn, bidPort, search=URIRef('%s%s' % (prefixes['mrs'], 'hasNetworkAddress')))
for item in out:
outtype = self.queryGraph(gIn, item, search=URIRef('%s%s' % (prefixes['mrs'], 'type')))
outval = self.queryGraph(gIn, item, search=URIRef('%s%s' % (prefixes['mrs'], 'value')))
if Literal('ipv4-address') in outtype:
if len(outval[0].split('/')) == 2:
output['hosts'][connInfo]['ip'] = str(outval[0])
# Now lets get service Info and what was requested.
out = self.queryGraph(gIn, bidPort, search=URIRef('%s%s' % (prefixes['nml'], 'hasService')))
output['hosts'][connInfo].setdefault('params', [])
serviceparams = {}
if out:
for key in ['availableCapacity', 'granularity', 'maximumCapacity',
'priority', 'reservableCapacity', 'type', 'unit']:
tmpout = self.queryGraph(gIn, out[0], search=URIRef('%s%s' % (prefixes['mrs'], key)))
if len(tmpout) >= 1:
serviceparams[key] = str(tmpout[0])
output['hosts'][connInfo]['params'].append(serviceparams)
returnout.append(output)
return returnout
def startwork(self):
""" Start Policy Service """
self.logger.info("=" * 80)
self.logger.info("Component PolicyService Started")
for siteName in self.config.get('general', 'sites').split(','):
workDir = self.config.get(siteName, 'privatedir') + "/PolicyService/"
createDirs(workDir)
# Committed to activating...
# committing, committed, activating, activated, remove, removing, cancel
dbobj = getVal(self.dbI, sitename=self.sitename)
for job in [['committing', self.stateMachine.committing],
['committed', self.stateMachine.committed],
['activating', self.stateMachine.activating],
['activated', self.stateMachine.activated],
['remove', self.stateMachine.remove],
['removing', self.stateMachine.removing],
['cancel', self.stateMachine.cancel],
['cancelConn', self.stateMachine.cancelledConnections]]:
self.logger.info("Starting check on %s deltas" % job[0])
job[1](dbobj)
def acceptDelta(self, deltapath):
""" Accept delta """
jOut = getAllHosts(self.sitename, self.logger)
fileContent = self.siteDB.getFileContentAsJson(deltapath)
os.unlink(deltapath) # File is not needed anymore.
toDict = dict(fileContent)
toDict["State"] = "accepting"
outputDict = {'addition': '', 'reduction': ''}
try:
self.logger.info(toDict["Content"])
self.logger.info(type(toDict["Content"]))
for key in ['reduction', 'addition']:
if key in toDict["Content"] and toDict["Content"][key]:
self.logger.info('Got Content %s for key %s', toDict["Content"][key], key)
tmpFile = tempfile.NamedTemporaryFile(delete=False)
try:
tmpFile.write(toDict["Content"][key])
except ValueError as ex:
self.logger.info('Received ValueError. More details %s. Try to write normally with decode', ex)
tmpFile.write(decodebase64(toDict["Content"][key]))
tmpFile.close()
outputDict[key] = self.parseDeltaRequest(tmpFile.name, jOut)
os.unlink(tmpFile.name)
except (IOError, KeyError, AttributeError, IndentationError, ValueError,
BadSyntax, HostNotFound, UnrecognizedDeltaOption) as ex:
outputDict = getError(ex)
dbobj = getVal(self.dbI, sitename=self.sitename)
if 'errorType' in outputDict.keys():
toDict["State"] = "failed"
toDict["Error"] = outputDict
toDict['ParsedDelta'] = {'addition': '', 'reduction': ''}
self.stateMachine.failed(dbobj, toDict)
else:
toDict["State"] = "accepted"
connID = None
for key in outputDict:
toDict["State"] = "accepted"
if not outputDict[key]:
continue
toDict['dtype'] = key
toDict['Type'] = key
self.logger.info('%s' % str(outputDict[key]))
toDict["ParsedDelta"] = outputDict
connID = []
for item in outputDict[key]:
connID.append(item['connectionID'])
toDict['ConnID'] = connID
toDict['modadd'] = 'idle'
self.stateMachine.accepted(dbobj, toDict)
# =================================
return toDict
class PolicyService(object):
""" Policy Service to accept deltas """
def __init__(self, config, logger):
self.logger = logger
self.config = config
self.siteDB = contentDB(logger=self.logger, config=self.config)
self.dbI = getDBConn()
self.stateMachine = StateMachine(self.logger)
def queryGraph(self, graphIn, sub=None, pre=None, obj=None, search=None):
""" Does search inside the graph based on provided parameters """
foundItems = []
for sIn, pIn, oIn in graphIn.triples((sub, pre, obj)):
if search:
if search == pIn:
self.logger.debug('Found item with search parameter')
self.logger.debug("s(subject) %s" % sIn)
self.logger.debug("p(predica) %s" % pIn)
self.logger.debug("o(object ) %s" % oIn)
self.logger.debug("-" * 50)
foundItems.append(oIn)
else:
self.logger.debug('Found item without search parameter')
self.logger.debug("s(subject) %s" % sIn)
self.logger.debug("p(predica) %s" % pIn)
self.logger.debug("o(object ) %s" % oIn)
self.logger.debug("-" * 50)
foundItems.append(oIn)
return foundItems
def getTimeScheduling(self, out, gIn, prefixes):
# This is for identifying LIFETIME! In case it fails to get correct timestamp,
# resources will be provisioned right away
# ======================================================
for timeline in out:
times = {}
for timev in ['end', 'start']:
tout = self.queryGraph(gIn,
timeline,
search=URIRef('%s%s' %
(prefixes['nml'], timev)))
temptime = None
try:
temptime = int(
time.mktime(parser.parse(str(tout[0])).timetuple()))
if time.daylight:
temptime -= 3600
except:
continue
times[timev] = temptime
if len(times.keys()) == 2:
return times
return {}
def parseDeltaRequest(self, inFileName, allKnownHosts, sitename):
"""Parse delta request to json"""
output = {}
self.logger.info("Parsing delta request %s ", inFileName)
prefixes = {}
prefixes['site'] = "%s:%s:%s" % (self.config.get(
'prefixes', 'site'), self.config.get(
sitename, 'domain'), self.config.get(sitename, 'year'))
prefixes['main'] = URIRef("%s:service+vsw" % prefixes['site'])
prefixes['nml'] = self.config.get('prefixes', 'nml')
prefixes['mrs'] = self.config.get('prefixes', 'mrs')
gIn = Graph()
gIn.parse(inFileName, format='turtle')
self.logger.info('Lets try to get connection ID subject')
connectionID = None
out = self.queryGraph(gIn, prefixes['main'])
if not out:
msg = 'Connection ID was not received. Something is w'
self.logger.info(msg)
return {}
if len(out) > 1:
msg = 'Received multiple connection IDs. Something is wrong...'
self.logger.info(msg)
return {}
output['connectionID'] = str(out[0])
connectionID = out[0]
self.logger.info('This is our connection ID: %s' % connectionID)
self.logger.info(
'Now lets get all info what it wants to do. Mainly bidPorts and labelSwapping flag'
)
bidPorts = self.queryGraph(
gIn,
connectionID,
search=URIRef('%s%s' % (prefixes['nml'], 'hasBidirectionalPort')))
out = self.queryGraph(gIn,
connectionID,
search=URIRef(
'%s%s' % (prefixes['nml'], 'labelSwapping')))
output['labelSwapping'] = str(out[0])
out = self.queryGraph(gIn,
connectionID,
search=URIRef('%s%s' %
(prefixes['nml'], 'existsDuring')))
out = self.getTimeScheduling(out, gIn, prefixes)
if len(out.keys()) == 2:
output['timestart'] = out['start']
output['timeend'] = out['end']
# =======================================================
self.logger.info(
'Now lets get all info for each bidirectionalPort, like vlan, ip, serviceInfo '
)
# We need mainly hasLabel, hasNetworkAddress
for bidPort in bidPorts:
# Get first which labels it has. # This provides us info about vlan tag
connInfo, output = getConnInfo(bidPort,
prefixes['site'],
output,
nostore=True)
print connInfo, allKnownHosts
if connInfo not in allKnownHosts:
print 'Ignore %s' % connInfo
continue
connInfo, output = getConnInfo(bidPort, prefixes['site'], output)
alias = self.queryGraph(gIn,
bidPort,
search=URIRef(
'%s%s' % (prefixes['nml'], 'isAlias')))
print alias, bidPorts
if alias and alias[0] not in bidPorts:
self.logger.info('Received alias for %s to %s' %
(bidPort, alias))
bidPorts.append(alias[0])
# Now let's get vlan ID
out = self.queryGraph(gIn,
bidPort,
search=URIRef('%s%s' %
(prefixes['nml'], 'hasLabel')))
if not out:
continue
out = self.queryGraph(gIn,
out[0],
search=URIRef('%s%s' %
(prefixes['nml'], 'value')))
output['hosts'][connInfo]['vlan'] = str(out[0])
# Now Let's get IP
out = self.queryGraph(
gIn,
bidPort,
search=URIRef('%s%s' % (prefixes['mrs'], 'hasNetworkAddress')))
if out:
out = self.queryGraph(gIn,
out[0],
search=URIRef(
'%s%s' % (prefixes['mrs'], 'value')))
output['hosts'][connInfo]['ip'] = str(out[0])
# Now lets get service Info and what was requested.
out = self.queryGraph(
gIn,
bidPort,
search=URIRef('%s%s' % (prefixes['nml'], 'hasService')))
output['hosts'][connInfo].setdefault('params', [])
serviceparams = {}
if out:
for key in [
'availableCapacity', 'granularity', 'maximumCapacity',
'priority', 'reservableCapacity', 'type', 'unit'
]:
print key
tmpout = self.queryGraph(
gIn,
out[0],
search=URIRef('%s%s' % (prefixes['mrs'], key)))
if len(tmpout) >= 1:
serviceparams[key] = str(tmpout[0])
output['hosts'][connInfo]['params'].append(serviceparams)
print output
return output
def reductionCompare(self, sitename, redID):
dbobj = getVal(self.dbI, sitename=sitename)
out = dbobj.get('deltas', search=[['connectionid', redID]])
if out:
return out[0]['uid']
return None
def startwork(self):
self.logger.info("=" * 80)
self.logger.info("Component PolicyService Started")
for siteName in self.config.get('general', 'sites').split(','):
workDir = self.config.get(siteName,
'privatedir') + "/PolicyService/"
createDirs(workDir)
self.logger.info('Working on Site %s' % siteName)
self.startworkmain(siteName)
def acceptDelta(self, deltapath, sitename):
jOut = getAllHosts(sitename, self.logger)
fileContent = self.siteDB.getFileContentAsJson(deltapath)
os.unlink(deltapath) # File is not needed anymore.
toDict = dict(fileContent)
toDict["State"] = "accepting"
outputDict = {'addition': '', 'reduction': ''}
try:
self.logger.info(toDict["Content"])
for key in ['reduction', 'addition']:
if key in toDict["Content"] and toDict["Content"][key]:
self.logger.info('Got Content %s for key %s',
toDict["Content"][key], key)
tmpFile = tempfile.NamedTemporaryFile(delete=False)
try:
tmpFile.write(toDict["Content"][key])
except ValueError as ex:
self.logger.info(
'Received ValueError. More details %s. Try to write normally with decode',
ex)
tmpFile.write(decodebase64(toDict["Content"][key]))
tmpFile.close()
outputDict[key] = self.parseDeltaRequest(
tmpFile.name, jOut, sitename)
self.logger.info("For %s this is delta location %s" %
(key, tmpFile.name))
# os.unlink(tmpFile.name)
except (IOError, KeyError, AttributeError, IndentationError,
ValueError, BadSyntax, HostNotFound,
UnrecognizedDeltaOption) as ex:
outputDict = getError(ex)
dbobj = getVal(self.dbI, sitename=sitename)
if 'errorType' in outputDict.keys():
toDict["State"] = "failed"
toDict["Error"] = outputDict
toDict['ParsedDelta'] = {'addition': '', 'reduction': ''}
self.stateMachine.failed(dbobj, toDict)
else:
toDict["State"] = "accepted"
toDict["ParsedDelta"] = outputDict
dtype = None
connID = None
for key in outputDict:
if not outputDict[key]:
continue
# If key is reduction. Find out which one.
# So this check will not be needed anymore.
dtype = key
connID = outputDict[key]['connectionID']
if key == 'reduction':
if "ReductionID" not in outputDict.keys():
self.logger.info('Trying to identify which to delete')
reductionIDMap = self.reductionCompare(
sitename, outputDict[key]['connectionID'])
toDict["ReductionID"] = reductionIDMap
else:
self.logger.info('ReductionID is already defined.')
toDict['Type'] = dtype
toDict['ConnID'] = connID
toDict['modadd'] = 'idle'
self.stateMachine.accepted(dbobj, toDict)
# =================================
return toDict
def startworkmain(self, sitename):
"""Main start """
# Committed to activating...
# committing, committed, activating, activated, remove, removing, cancel
dbobj = getVal(self.dbI, sitename=sitename)
for job in [['committing', self.stateMachine.committing],
['committed', self.stateMachine.committed],
['activating', self.stateMachine.activating],
['activated', self.stateMachine.activated],
['remove', self.stateMachine.remove],
['removing', self.stateMachine.removing],
['cancel', self.stateMachine.cancel]]:
self.logger.info("Starting check on %s deltas" % job[0])
job[1](dbobj)