def addhost(self, inputDict, **kwargs): """Adding new host to DB. Must provide dictionary with: hostname -> hostname of new host ip -> ip of new host lat -> latitude, enought to provide approximately site location lon -> longitude, enought to provide approximately site location Examples: GOOD: {"hostname": "dtn-rm.ultralight.org", "ip": "1.2.3.4"} """ dbobj = getVal(self.dbI, **kwargs) host = dbobj.get('hosts', limit=1, search=[['ip', inputDict['ip']]]) if not host: out = { 'hostname': inputDict['hostname'], 'ip': inputDict['ip'], 'insertdate': inputDict['insertTime'], 'updatedate': inputDict['updateTime'], 'hostinfo': str(inputDict) } dbobj.insert('hosts', [out]) else: print 'This host is already in db. Why to add several times?' raise BadRequestError( 'This host is already in db. Why to add several times?') return
def acceptDelta(self, deltapath): """Accept delta.""" jOut = getAllHosts(self.sitename, self.logger) fileContent = self.siteDB.getFileContentAsJson(deltapath) os.unlink(deltapath) # File is not needed anymore. toDict = dict(fileContent) toDict["State"] = "accepting" outputDict = {'addition': '', 'reduction': ''} try: self.logger.info(toDict["Content"]) self.logger.info(type(toDict["Content"])) for key in ['reduction', 'addition']: if key in toDict["Content"] and toDict["Content"][key]: self.logger.info('Got Content %s for key %s', toDict["Content"][key], key) tmpFile = tempfile.NamedTemporaryFile(delete=False, mode="w+") try: tmpFile.write(toDict["Content"][key]) except ValueError as ex: self.logger.info( 'Received ValueError. More details %s. Try to write normally with decode', ex) tmpFile.write(decodebase64(toDict["Content"][key])) tmpFile.close() outputDict[key] = self.parseDeltaRequest( tmpFile.name, jOut) os.unlink(tmpFile.name) except (IOError, KeyError, AttributeError, IndentationError, ValueError, BadSyntax, HostNotFound, UnrecognizedDeltaOption) as ex: outputDict = getError(ex) dbobj = getVal(self.dbI, sitename=self.sitename) if 'errorType' in list(outputDict.keys()): toDict["State"] = "failed" toDict["Error"] = outputDict toDict['ParsedDelta'] = {'addition': '', 'reduction': ''} self.stateMachine.failed(dbobj, toDict) else: toDict["State"] = "accepted" connID = [] toDict["ParsedDelta"] = outputDict toDict['modadd'] = 'idle' for key in outputDict: if not outputDict[key]: continue toDict['Type'] = 'modify' if 'Type' in toDict.keys() else key # In case of modify, only addition connection IDs are stored; # otherwise, corresponding type connectionIDs if toDict['Type'] == 'modify': connID = [] for item in outputDict['addition']: connID.append(item['connectionID']) else: for item in outputDict[key]: connID.append(item['connectionID']) toDict['ConnID'] = connID self.stateMachine.accepted(dbobj, toDict) # ================================= return toDict
def reportServiceStatus(servicename, status, sitename, logger, hostname=""): """ Report service state to DB """ try: if not hostname: hostname = socket.gethostname() dbOut = { 'hostname': hostname, 'servicestate': status, 'servicename': servicename, 'updatedate': getUTCnow() } dbI = getDBConn(servicename) dbobj = getVal(dbI, **{'sitename': sitename}) services = dbobj.get('servicestates', search=[['hostname', hostname], ['servicename', servicename]]) if not services: dbobj.insert('servicestates', [dbOut]) else: dbobj.update('servicestates', [dbOut]) except Exception: excType, excValue = sys.exc_info()[:2] logger.critical( "Error details in reportServiceStatus. ErrorType: %s, ErrMsg: %s", str(excType.__name__), excValue)
def getalldebughostname(self, **kwargs): dbobj = getVal(self.dbI, **kwargs) search = [['hostname', kwargs['mReg'][1]], ['state', 'new']] return dbobj.get('debugrequests', orderby=['updatedate', 'DESC'], search=search, limit=1000)
def getdeltastates(self, deltaID, **kwargs): """Get delta states from database.""" dbobj = getVal(self.dbI, **kwargs) out = dbobj.get('states', search=[['deltaid', deltaID]]) if not out: raise DeltaNotFound( "Delta with %s id was not found in the system" % deltaID) return out
def deleteAll(sitename, deltaUID=None): """delete all deltas.""" dbI = getDBConn('cancelalldeltas') dbobj = getVal(dbI, sitename=sitename) for delta in dbobj.get('deltas'): if deltaUID and delta['uid'] != deltaUID: continue print('Cancel %s' % delta['uid']) STATEMACHINE._stateChangerDelta(dbobj, 'remove', **delta)
def getdelta(self, deltaID=None, **kwargs): """ Get delta from file """ dbobj = getVal(self.dbI, **kwargs) if not deltaID: return dbobj.get('deltas') out = dbobj.get('deltas', search=[['uid', deltaID]]) if not out: raise DeltaNotFound("Delta with %s id was not found in the system" % deltaID) return out[0]
def updatedebug(self, inputDict, **kwargs): """Update debug action information.""" dbobj = getVal(self.dbI, **kwargs) out = { 'id': kwargs['mReg'][1], 'state': inputDict['state'], 'output': inputDict['output'], 'updatedate': getUTCnow() } return dbobj.update('debugrequests', [out])
def getdebug(self, **kwargs): """Get Debug action for specific ID.""" dbobj = getVal(self.dbI, **kwargs) search = None if kwargs['mReg'][1] != 'ALL': search = [['id', kwargs['mReg'][1]]] return dbobj.get('debugrequests', orderby=['insertdate', 'DESC'], search=search, limit=1000)
def getmodel(self, modelID=None, content=False, **kwargs): """ Get all models """ dbobj = getVal(self.dbI, **kwargs) if not modelID: return dbobj.get('models', orderby=['insertdate', 'DESC']) model = dbobj.get('models', limit=1, search=[['uid', modelID]]) if not model: raise ModelNotFound("Model with %s id was not found in the system" % modelID) if content: return getAllFileContent(model[0]['fileloc']) return model[0]
def submitdebug(self, inputDict, **kwargs): """Submit new debug action request.""" dbobj = getVal(self.dbI, **kwargs) out = { 'hostname': inputDict['dtn'], 'state': 'new', 'requestdict': json.dumps(inputDict), 'output': '', 'insertdate': getUTCnow(), 'updatedate': getUTCnow() } return dbobj.insert('debugrequests', [out])
def startworkmain(self, sitename): """Main start """ # Committed to activating... # committing, committed, activating, activated, remove, removing, cancel dbobj = getVal(self.dbI, sitename=sitename) for job in [['committing', self.stateMachine.committing], ['committed', self.stateMachine.committed], ['activating', self.stateMachine.activating], ['activated', self.stateMachine.activated], ['remove', self.stateMachine.remove], ['removing', self.stateMachine.removing], ['cancel', self.stateMachine.cancel]]: self.logger.info("Starting check on %s deltas" % job[0]) job[1](dbobj)
def addNewDelta(self, uploadContent, environ, **kwargs): """ Add new delta """ dbobj = getVal(self.dbI, **kwargs) hashNum = uploadContent['id'] if dbobj.get('deltas', search=[['uid', hashNum]], limit=1): # This needs to be supported as it can be re-initiated again. TODO msg = 'Something weird has happened... Check server logs; Same ID is already in DB' kwargs['http_respond'].ret_409('application/json', kwargs['start_response'], None) return getCustomOutMsg(errMsg=msg, errCode=409) tmpfd = NamedTemporaryFile(delete=False) tmpfd.close() self.getmodel(uploadContent['modelId'], **kwargs) outContent = { "ID": hashNum, "InsertTime": getUTCnow(), "UpdateTime": getUTCnow(), "Content": uploadContent, "State": "accepting", "modelId": uploadContent['modelId'] } self.siteDB.saveContent(tmpfd.name, outContent) out = self.policer[kwargs['sitename']].acceptDelta(tmpfd.name) outDict = { 'id': hashNum, 'lastModified': convertTSToDatetime(outContent['UpdateTime']), 'href': "%s/%s" % (environ['SCRIPT_URI'], hashNum), 'modelId': out['modelId'], 'state': out['State'], 'reduction': out['ParsedDelta']['reduction'], 'addition': out['ParsedDelta']['addition'] } print 'Delta was %s. Returning info %s' % (out['State'], outDict) if out['State'] in ['accepted']: kwargs['http_respond'].ret_201( 'application/json', kwargs['start_response'], [('Last-Modified', httpdate(out['UpdateTime'])), ('Location', outDict['href'])]) return outDict else: kwargs['http_respond'].ret_500('application/json', kwargs['start_response'], None) if 'Error' in out.keys(): errMsg = "" for key in ['errorNo', 'errorType', 'errMsg']: if key in out['Error'].keys(): errMsg += " %s: %s" % (key, out['Error'][key]) return getCustomOutMsg(errMsg=errMsg, exitCode=500)
def getdeltaAll(sitename): dbI = getDBConn('listalldeltas') dbobj = getVal(dbI, sitename=sitename) for delta in dbobj.get('deltas'): delta['addition'] = evaldict(delta['addition']) delta['reduction'] = evaldict(delta['reduction']) print '='*80 print 'Delta UID : ', delta['uid'] print 'Delta RedID: ', delta['reductionid'] print 'Delta State: ', delta['state'] print 'Delta ModAdd: ', delta['modadd'] print 'Delta InsDate:', delta['insertdate'] print 'Delta Update: ', delta['updatedate'] print 'Delta Model: ', delta['modelid'] print 'Delta connID: ', delta['connectionid'] print 'Delta Deltatype: ', delta['deltat'] print '-'*20 print 'Delta times' for deltatimes in dbobj.get('states', search=[['deltaid', delta['uid']]]): print 'State: %s Date: %s' % (deltatimes['state'], deltatimes['insertdate']) if delta['deltat'] not in ['reduction', 'addition']: print 'SOMETHING WRONG WITH THIS DELTA. It does not have any type defined. Was not parsed properly' continue if not isinstance(delta[delta['deltat']], list): conns = [delta[delta['deltat']]] else: conns = delta[delta['deltat']] for conn in conns: if 'hosts' not in conn.keys(): print 'SOMETHING WRONG WITH THIS DELTA. It does not have any hosts defined.' continue for hostname in conn['hosts'].keys(): print '-'*20 print 'Host States %s' % hostname for hoststate in dbobj.get('hoststates', search=[['deltaid', delta['uid']], ['hostname', hostname]]): print 'Host %s State %s' % (hostname, hoststate['state']) print 'Insertdate %s UpdateDate %s' % (hoststate['insertdate'], hoststate['updatedate']) print '-'*20 print 'Host State History' for hstatehistory in dbobj.get('hoststateshistory', search=[['deltaid', delta['uid']], ['hostname', hostname]]): print 'State: %s, Date: %s' % (hstatehistory['state'], hstatehistory['insertdate']) print '-'*20 print 'Connection details' for conn in conns: for dConn in dbobj.get('delta_connections', search=[['connectionid', conn['connectionID']]]): print dConn
def commitdelta(self, deltaID, newState='UNKNOWN', internal=False, hostname=None, **kwargs): """Change delta state.""" dbobj = getVal(self.dbI, **kwargs) if internal: out = dbobj.get('hoststates', search=[['deltaid', deltaID], ['hostname', hostname]]) if not out: msg = 'This query did not returned any host states for %s %s' % ( deltaID, hostname) raise WrongDeltaStatusTransition(msg) self.stateM._stateChangerHost( dbobj, out[0]['id'], **{ 'deltaid': deltaID, 'state': newState, 'insertdate': getUTCnow(), 'hostname': hostname }) if newState == 'remove': # Remove state comes only from modify and by agent itself self.stateM.stateChange(dbobj, { 'uid': deltaID, 'state': newState }) return getCustomOutMsg(msg='Internal State change approved', exitCode=200) delta = self.getdelta(deltaID, **kwargs) print('Commit Action for delta %s' % delta) # Now we go directly to commited in case of commit if delta['state'] != 'accepted': msg = "Delta state in the system is not in accepted state. \ State on the system: %s. Not allowed to change." % delta[ 'state'] print(msg) raise WrongDeltaStatusTransition(msg) self.stateM.commit(dbobj, {'uid': deltaID, 'state': 'committing'}) return {'status': 'OK'}
def startwork(self): """ Start Policy Service """ self.logger.info("=" * 80) self.logger.info("Component PolicyService Started") for siteName in self.config.get('general', 'sites').split(','): workDir = self.config.get(siteName, 'privatedir') + "/PolicyService/" createDirs(workDir) # Committed to activating... # committing, committed, activating, activated, remove, removing, cancel dbobj = getVal(self.dbI, sitename=self.sitename) for job in [['committing', self.stateMachine.committing], ['committed', self.stateMachine.committed], ['activating', self.stateMachine.activating], ['activated', self.stateMachine.activated], ['remove', self.stateMachine.remove], ['removing', self.stateMachine.removing], ['cancel', self.stateMachine.cancel], ['cancelConn', self.stateMachine.cancelledConnections]]: self.logger.info("Starting check on %s deltas" % job[0]) job[1](dbobj)
def getdeltainfo(sitename, deltaUID): """Get all delta information. INPUT: sitename - str mandatory deltaUID - str mandatory """ dbI = getDBConn('acceptdelta') dbobj = getVal(dbI, sitename=sitename) for delta in dbobj.get('deltas'): if delta['uid'] != deltaUID: continue delta['addition'] = evaldict(delta['addition']) delta['reduction'] = evaldict(delta['reduction']) LOGGER.info('=' * 80) LOGGER.info('Delta UID : %s', delta['uid']) LOGGER.info('Delta RedID: %s', delta['reductionid']) LOGGER.info('Delta State: %s', delta['state']) LOGGER.info('Delta ModAdd: %s', delta['modadd']) LOGGER.info('Delta InsDate: %s', delta['insertdate']) LOGGER.info('Delta Update: %s', delta['updatedate']) LOGGER.info('Delta Model: %s', delta['modelid']) LOGGER.info('Delta connID: %s', delta['connectionid']) LOGGER.info('Delta Deltatype: %s', delta['deltat']) LOGGER.info('-' * 20) LOGGER.info('Delta times') for deltatimes in dbobj.get('states', search=[['deltaid', delta['uid']]]): LOGGER.info('State: %s Date: %s', deltatimes['state'], deltatimes['insertdate']) if delta['deltat'] in ['reduction', 'addition']: for hostname in list(delta[delta['deltat']]['hosts'].keys()): LOGGER.info('-' * 20) LOGGER.info('Host States %s', hostname) for hoststate in dbobj.get('hoststates', search=[['deltaid', delta['uid']], ['hostname', hostname]]): LOGGER.info('Host %s State %s', hostname, hoststate['state']) LOGGER.info('Insertdate %s UpdateDate %s', hoststate['insertdate'], hoststate['updatedate']) LOGGER.info('-' * 20) LOGGER.info('Host State History') for hstatehistory in dbobj.get('hoststateshistory', search=[['deltaid', delta['uid']], ['hostname', hostname]]): LOGGER.info('State: %s, Date: %s', hstatehistory['state'], hstatehistory['insertdate']) return delta, dbobj
def updatehost(self, inputDict, **kwargs): """ Update Host in DB. Must provide dictionary with: ip -> ip of new host Example: GOOD: {"ip": "1.2.3.4", "site": "T2_US_Caltech", "status": "benchhmark", "hostype": "gridftp", "port": 11234}""" # Validate that these entries are known... dbobj = getVal(self.dbI, **kwargs) host = dbobj.get('hosts', limit=1, search=[['ip', inputDict['ip']]]) if not host: raise NotFoundError( 'This IP %s is not registered at all. Call addhost' % inputDict['ip']) out = { 'id': host[0]['id'], 'hostname': inputDict['hostname'], 'ip': inputDict['ip'], 'updatedate': getUTCnow(), 'hostinfo': str(inputDict) } dbobj.update('hosts', [out]) return
def getHostNameIDs(self, hostname, state, **kwargs): """ Get Hostname IDs """ dbobj = getVal(self.dbI, **kwargs) return dbobj.get('hoststates', search=[['hostname', hostname], ['state', state]])
def getdeltaAll(sitename, deltaUID): dbI = getDBConn('analyzedelta') dbobj = getVal(dbI, sitename=sitename) policer = polS.PolicyService(CONFIG, LOGGER) for delta in dbobj.get('deltas'): if delta['uid'] != deltaUID: continue delta['addition'] = evaldict(delta['addition']) delta['reduction'] = evaldict(delta['reduction']) print('=' * 80) print('Delta UID : ', delta['uid']) print('Delta RedID: ', delta['reductionid']) print('Delta State: ', delta['state']) print('Delta ModAdd: ', delta['modadd']) print('Delta InsDate:', delta['insertdate']) print('Delta Update: ', delta['updatedate']) print('Delta Model: ', delta['modelid']) print('Delta connID: ', delta['connectionid']) print('Delta Deltatype: ', delta['deltat']) print('-' * 20) import pprint pprint.pprint(delta) print('Delta times') for deltatimes in dbobj.get('states', search=[['deltaid', delta['uid']]]): print('State: %s Date: %s' % (deltatimes['state'], deltatimes['insertdate'])) if delta['deltat'] in ['reduction', 'addition']: for hostname in list(delta[delta['deltat']]['hosts'].keys()): print('-' * 20) print('Host States %s' % hostname) for hoststate in dbobj.get('hoststates', search=[['deltaid', delta['uid']], ['hostname', hostname]]): print('Host %s State %s' % (hostname, hoststate['state'])) print('Insertdate %s UpdateDate %s' % (hoststate['insertdate'], hoststate['updatedate'])) print('-' * 20) print('Host State History') for hstatehistory in dbobj.get( 'hoststateshistory', search=[['deltaid', delta['uid']], ['hostname', hostname]]): print('State: %s, Date: %s' % (hstatehistory['state'], hstatehistory['insertdate'])) toDict = ast.literal_eval(str(delta['content'])) jOut = getAllHosts(sitename, LOGGER) for key in ['reduction', 'addition']: print(list(toDict.keys())) if key in toDict and toDict[key]: print('Got Content %s for key %s', toDict[key], key) tmpFile = tempfile.NamedTemporaryFile(delete=False, mode="w+") try: tmpFile.write(toDict[key]) except ValueError as ex: print( 'Received ValueError. More details %s. Try to write normally with decode', ex) tmpFile.write(decodebase64(toDict["Content"][key])) tmpFile.close() # outputDict[key] = self.parseDeltaRequest(tmpFile.name, jOut) print("For %s this is delta location %s" % (key, tmpFile.name)) out = policer.parseDeltaRequest(tmpFile.name, jOut) if not out: out = policer.parseDeltaRequest(tmpFile.name, jOut, sitename) print(out)
def startwork(self): """Main start """ self.logger.info('Started LookupService work') dbObj = getVal(self.dbI, **{'sitename': self.sitename}) workDir = self.config.get(self.sitename, 'privatedir') + "/LookUpService/" createDirs(workDir) self.newGraph = Graph() jOut = getAllHosts(self.sitename, self.logger) # ================================================================================== # 1. Define Basic MRML Prefixes # ================================================================================== self.defineMRMLPrefixes() # ================================================================================== # 2. Define Basic MRML Definition # ================================================================================== self.defineMRMLServices() self.hosts = {} for _, nodeDict in jOut.items(): # ================================================================================== # 3. Define Node inside yaml # ================================================================================== self.defineNodeInformation(nodeDict) # ================================================================================== # 4. Define Routing Service information # ================================================================================== self.defineLayer3MRML(nodeDict) # ================================================================================== # 5. Define Host Information and all it's interfaces. # ================================================================================== self.defineHostInfo(nodeDict) # ================================================================================== # 6. Define Switch information from Switch Lookup Plugin # ================================================================================== self.addSwitchInfo(jOut) saveName = self.getModelSavePath() with open(saveName, "w") as fd: fd.write(self.newGraph.serialize(format='turtle')) hashNum = generateHash(self.newGraph.serialize(format='turtle')) # Append all deltas to the model self.appendDeltas(dbObj, saveName) if dbObj.get('models', limit=1, search=[['uid', hashNum]]): raise Exception('hashNum %s is already in database...' % hashNum) self.logger.info('Checking if new model is different from previous') modelsEqual, modelinDB = self.checkForModelDiff(dbObj, saveName) lastKnownModel = {'uid': hashNum, 'insertdate': getUTCnow(), 'fileloc': saveName, 'content': str(self.newGraph.serialize(format='turtle'))} if modelsEqual: if modelinDB[0]['insertdate'] < int(getUTCnow() - 3600): # Force to update model every hour, Even there is no update; self.logger.info('Forcefully update model in db as it is older than 1h') dbObj.insert('models', [lastKnownModel]) else: self.logger.info('Models are equal.') lastKnownModel = modelinDB[0] os.unlink(saveName) else: self.logger.info('Models are different. Update DB') dbObj.insert('models', [lastKnownModel]) self.logger.debug('Last Known Model: %s' % str(lastKnownModel)) # Clean Up old models (older than 24h.) for model in dbObj.get('models', limit=100, orderby=['insertdate', 'ASC']): if model['insertdate'] < int(getUTCnow() - 86400): self.logger.debug('delete %s', model) try: os.unlink(model['fileloc']) except OSError as ex: self.logger.debug('Got OS Error removing this model %s. Exc: %s' % (model, str(ex))) dbObj.delete('models', [['id', model['id']]])
def reductionCompare(self, sitename, redID): dbobj = getVal(self.dbI, sitename=sitename) out = dbobj.get('deltas', search=[['connectionid', redID]]) if out: return out[0]['uid'] return None
def getdata(self, **kwargs): """ Return all available Hosts data, where key is IP address """ dbobj = getVal(self.dbI, **kwargs) return dbobj.get('hosts', orderby=['updatedate', 'DESC'], limit=1000)
def acceptDelta(self, deltapath, sitename): jOut = getAllHosts(sitename, self.logger) fileContent = self.siteDB.getFileContentAsJson(deltapath) os.unlink(deltapath) # File is not needed anymore. toDict = dict(fileContent) toDict["State"] = "accepting" outputDict = {'addition': '', 'reduction': ''} try: self.logger.info(toDict["Content"]) for key in ['reduction', 'addition']: if key in toDict["Content"] and toDict["Content"][key]: self.logger.info('Got Content %s for key %s', toDict["Content"][key], key) tmpFile = tempfile.NamedTemporaryFile(delete=False) try: tmpFile.write(toDict["Content"][key]) except ValueError as ex: self.logger.info( 'Received ValueError. More details %s. Try to write normally with decode', ex) tmpFile.write(decodebase64(toDict["Content"][key])) tmpFile.close() outputDict[key] = self.parseDeltaRequest( tmpFile.name, jOut, sitename) self.logger.info("For %s this is delta location %s" % (key, tmpFile.name)) # os.unlink(tmpFile.name) except (IOError, KeyError, AttributeError, IndentationError, ValueError, BadSyntax, HostNotFound, UnrecognizedDeltaOption) as ex: outputDict = getError(ex) dbobj = getVal(self.dbI, sitename=sitename) if 'errorType' in outputDict.keys(): toDict["State"] = "failed" toDict["Error"] = outputDict toDict['ParsedDelta'] = {'addition': '', 'reduction': ''} self.stateMachine.failed(dbobj, toDict) else: toDict["State"] = "accepted" toDict["ParsedDelta"] = outputDict dtype = None connID = None for key in outputDict: if not outputDict[key]: continue # If key is reduction. Find out which one. # So this check will not be needed anymore. dtype = key connID = outputDict[key]['connectionID'] if key == 'reduction': if "ReductionID" not in outputDict.keys(): self.logger.info('Trying to identify which to delete') reductionIDMap = self.reductionCompare( sitename, outputDict[key]['connectionID']) toDict["ReductionID"] = reductionIDMap else: self.logger.info('ReductionID is already defined.') toDict['Type'] = dtype toDict['ConnID'] = connID toDict['modadd'] = 'idle' self.stateMachine.accepted(dbobj, toDict) # ================================= return toDict
def getalldebugids(self, **kwargs): """Get All Debug IDs.""" dbobj = getVal(self.dbI, **kwargs) return dbobj.get('debugrequestsids', orderby=['updatedate', 'DESC'], limit=1000)