def notifyUpdate(self):
"""
Notify user
"""
# update context and rns of all connected users
data = ('context-server', ('update', self.context.getInformations()))
ESI.instance().notifyAll(body=data)
def deleteBackups(self):
"""
Delete all backups
@return:
@rtype:
"""
ret = self.context.CODE_ERROR
try:
# delete all files
files = os.listdir(self.destBackup)
for x in files:
fullpath = os.path.join(self.destBackup, x)
if os.path.isfile(fullpath):
os.remove(fullpath)
else:
shutil.rmtree(fullpath)
# update all connected admin users
notif = {}
notif['repo-adapters'] = {}
data = ('repositories', ('reset', notif))
ESI.instance().notifyAll(body=data)
return self.context.CODE_OK
except OSError as e:
self.trace(e)
return self.context.CODE_FORBIDDEN
except Exception as e:
raise Exception(e)
return ret
return ret
def refreshTestEnvironment(self):
"""
Refresh test environment
And notify all connected user
"""
for user_login, user_profile in self.usersConnected.items():
if user_profile['profile']['administrator'] or user_profile['profile']['tester'] or \
user_profile[cur_user]['profile']['developer'] :
data = ('context-server',
("update", self.getInformations(user=user_login)))
ESI.instance().notify(body=data, toUser=user_login)
return True
def zipDataV2(self, dirToday, dirTest, destPathZip, replayId, projectId=0, virtualName = ""):
"""
Zip data by adapters and notify users in just one zip
@param dirToday: date YYYY-MM-DD
@type dirToday: string
@param dirTest:
@type dirTest: string
@param destPathZip:
@type destPathZip: string
@param replayId:
@type replayId: string
"""
self.trace("Starting to zip all adapters logs")
ret = False
try:
mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday)
testDir = "%s/%s" % (mainDir, dirTest)
# prepare the file name
tp = time.strftime( "%Y-%m-%d_%H-%M-%S", time.localtime(time.time()) ) \
+ ".%3.3d" % int((time.time() * 1000) % 1000 )
fileName = "%s_%s_%s" % (self.prefixAdaptersAll, tp , replayId)
# zip the folder
zipped = self.zipFolder(folderPath=testDir, zipName="%s.zip" % fileName, zipPath=destPathZip)
if zipped == self.context.CODE_OK:
# notify users
if Settings.getInt( 'Notifications', 'archives'):
size_ = os.path.getsize( "%s/%s.zip" % (destPathZip, fileName) )
notif = {}
m = [ { "type": "folder", "name": dirToday, "project": "%s" % projectId,
"content": [ { "type": "folder", "name": dirTest, "project": "%s" % projectId, "virtual-name": virtualName,
"content": [ {"type": "file", "name": "%s.zip" % fileName, 'size': str(size_), "project": "%s" % projectId } ]} ] } ]
notif['archive'] = m
data = ( 'archive', ( None, notif) )
ESI.instance().notifyByUserTypes(body = data,
admin=True,
monitor=False,
tester=True)
ret = True
else:
self.error( 'error to zip data adapters' )
ret = False
except Exception as e:
self.error( 'unable to zip data adapters v2: %s' % str(e) )
return ret
def addDefaultProbe(self, pName, pType, pDescr):
"""
Add default probe
@type pName:
@param pName:
@type pType:
@param pType:
@type pDescr:
@param pDescr:
@return:
@rtype: boolean
"""
ret = self.context.CODE_ERROR
try:
if self.configsFile is not None:
# add the section in the config file object
self.configsFile.add_section(pName)
self.configsFile.set(pName, 'enable', 1)
self.configsFile.set(pName, 'type', pType)
self.configsFile.set(pName, 'description', pDescr)
# write date the file
f = open("%s/probes.ini" % Settings.getDirExec(), 'w')
self.configsFile.write(f)
f.close()
# notify all admin and tester
notif = ('probes-default', ('add', self.getDefaultProbes()))
ESI.instance().notifyByUserTypes(body=notif,
admin=True,
monitor=False,
tester=True)
# return OK
ret = self.context.CODE_OK
except ConfigParser.DuplicateSectionError:
self.error("probe already exist %s" % str(pName))
ret = self.context.CODE_ALLREADY_EXISTS
except Exception as e:
self.error("unable to add default probe: %s" % str(e))
ret = self.context.CODE_FAILED
return ret
def delDefaultProbe(self, pName):
"""
Delete a default probe
@type pName:
@param pName:
@return:
@rtype: boolean
"""
ret = self.context.CODE_ERROR
try:
if self.configsFile is not None:
# remove the section in the config file object
self.configsFile.remove_section(pName)
# write date the file
f = open("%s/probes.ini" % Settings.getDirExec(), 'w')
self.configsFile.write(f)
f.close()
# notify all admin and tester
notif = ('probes-default', ('del', self.getDefaultProbes()))
ESI.instance().notifyByUserTypes(body=notif,
admin=True,
monitor=False,
tester=True)
runningProbe = PSI.instance().getProbe(pname=pName)
if runningProbe is not None:
runningProbe['auto-startup'] = False
notif2 = ('probes', ('del', PSI.instance().getProbes()))
ESI.instance().notifyByUserTypes(body=notif2,
admin=True,
monitor=False,
tester=True)
# return OK
ret = self.context.CODE_OK
except ConfigParser.NoSectionError:
self.error("probe not found: %s" % str(pName))
ret = self.context.CODE_NOT_FOUND
except Exception as e:
self.error("unable to delete default probe: %s" % str(e))
ret = self.context.CODE_FAILED
return ret
def unregisterChannelUser(self, login):
"""
Force channel disconnection
"""
self.info("Unregister user Login=%s" % login)
UsersManager.instance().setOnlineStatus(login=login, online=False)
if not login in self.usersConnected:
self.trace("unregister user from api, user %s not found" % login)
return self.CODE_NOT_FOUND
else:
userProfile = self.usersConnected[login]
# close the network link with the client if exists
if userProfile['address'] in ESI.instance().clients:
ESI.instance().stopClient(client=userProfile['address'])
else:
user_removed = self.usersConnected.pop(login)
del user_removed
return self.CODE_OK
def addResultScript(self, scriptResult, scriptUser, scriptDuration,
scriptProject):
"""
Add script result to db
@param scriptResult: COMPLETE, ERROR, KILLED, etc...
@type scriptResult: str
@param scriptUser: username
@type scriptUser: str
@param scriptDuration: duration
@type scriptDuration: str
"""
if not Settings.getInt('MySql', 'insert-test-statistics'):
return
self.trace('add result sc %s' % scriptResult)
try:
strDate = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(time.time()))
#
ret, rows = DbManager.instance().querySQL(
query=
"INSERT INTO `%s` (date, result, user_id, duration, project_id) VALUES ('%s','%s',%s,'%s','%s')"
% (self.dbt_scripts, strDate, scriptResult, scriptUser,
scriptDuration, scriptProject))
if not ret:
raise Exception("failed to add result script in db")
except Exception as e:
self.error(e)
else:
if self.notifyUsers:
data = ('stats', (None, self.getStats()))
ESI.instance().notifyByUserTypes(body=data,
admin=True,
monitor=True,
tester=False)
def createBackup(self, backupName):
"""
Create a backup of all adapters
@type backupName:
@param backupName:
@return:
@rtype:
"""
ret = self.context.CODE_ERROR
try:
backupIndex = self.getLastBackupIndex(pathBackups=self.destBackup)
backupDate = self.getTimestamp()
backupFilename = '%s%s_%s_%s' % (self.prefixBackup, backupIndex,
backupName, backupDate)
# new in v14.0.0: create tar gz
if Settings.getInt('Backups', 'adapters-dest-tar-gz'):
self.trace("backup adapters to %s/%s.tar.gz" %
(self.destBackup, backupFilename))
DEVNULL = open(os.devnull, 'w')
__cmd__ = "%s cvfz %s/%s.tar.gz -C %s ." % (Settings.get(
'Bin',
'tar'), self.destBackup, backupFilename, self.testsPath)
ret = subprocess.call(__cmd__,
shell=True,
stdout=DEVNULL,
stderr=DEVNULL)
if ret: raise Exception("unable to tar sut adapter pkg")
ret = self.context.CODE_OK
# create a zip file
if Settings.getInt('Backups', 'adapters-dest-zip'):
self.trace("backup adapters to %s/%s.zip" %
(self.destBackup, backupFilename))
zipped = self.zipFolder(folderPath=self.testsPath,
zipName="%s.zip" % backupFilename,
zipPath=self.destBackup,
ignoreExt=['.pyc', '.pyo'])
ret = zipped
if zipped == self.context.CODE_OK:
self.info("backup adapters successfull: %s" %
backupFilename)
# now notify all connected admin users
backupSize = os.path.getsize(
"%s/%s.zip" % (self.destBackup, backupFilename))
notif = {}
notif['repo-adapters'] = {}
notif['repo-adapters']['backup'] = {
'name': backupName,
'date': backupDate,
'size': backupSize,
'fullname': "%s.zip" % backupFilename
}
data = ('repositories', (None, notif))
ESI.instance().notifyAllAdmins(body=data)
else:
self.error("backup adapters %s failed" % backupFilename)
except Exception as e:
raise Exception("[createBackup] %s" % str(e))
return ret
def cleanup(self):
"""
Cleanup the server
"""
self.info('Cleanup...')
self.trace("finalize probes manager")
try:
ProbesManager.finalize()
except Exception:
pass
self.trace("finalize agent manager")
try:
AgentsManager.finalize()
except Exception:
pass
self.trace("finalize toolbox manager")
try:
ToolboxManager.finalize()
except Exception:
pass
self.trace("finalize settings")
try:
Settings.finalize()
except Exception:
pass
self.trace("finalize context")
try:
Context.finalize()
except Exception:
pass
self.trace("finalize projects manager")
try:
ProjectsManager.finalize()
except Exception:
pass
self.trace("finalize users manager")
try:
UsersManager.finalize()
except Exception:
pass
self.trace("finalize stats manager")
try:
StatsManager.finalize()
except Exception:
pass
self.trace("finalize task manager")
try:
TaskManager.finalize()
except Exception:
pass
self.trace("finalize test public manager")
try:
RepoPublic.finalize()
except Exception:
pass
self.trace("finalize test repo manager")
try:
RepoTests.finalize()
except Exception:
pass
self.trace("finalize test archives manager")
try:
RepoArchives.finalize()
except Exception:
pass
self.trace("finalize helper manager")
try:
HelperManager.finalize()
except Exception:
pass
self.trace("finalize libraries manager")
try:
RepoLibraries.finalize()
except Exception:
pass
self.trace("finalize adapters manager")
try:
RepoAdapters.finalize()
except Exception:
pass
self.trace("finalize adapters data storage")
try:
StorageDataAdapters.finalize()
except Exception:
pass
self.trace("finalize WSU")
try:
RestServerInterface.instance().stop()
RestServerInterface.finalize()
except Exception:
pass
self.trace("finalize ESI")
try:
EventServerInterface.instance().stopSA()
EventServerInterface.finalize()
except Exception:
pass
self.trace("finalize TSI")
try:
TestServerInterface.instance().stopSA()
TestServerInterface.finalize()
except Exception:
pass
self.trace("finalize PSI")
try:
ProbeServerInterface.instance().stopSA()
ProbeServerInterface.finalize()
except Exception:
pass
self.trace("finalize ASI")
try:
AgentServerInterface.instance().stopSA()
AgentServerInterface.finalize()
except Exception:
pass
self.trace("finalize db manager")
try:
DbManager.finalize()
except Exception:
pass
self.trace("finalize logger, cli")
try:
CliFunctions.finalize()
Logger.finalize()
except Exception:
pass
def initialize(self):
"""
Starts all modules
Exit if the service is alreayd running or if the config file is missing
"""
starttime = time.time()
if self.isrunning():
sys.stdout.write(" (server is already running)")
sys.exit(1)
self.daemonize()
try:
# Initialize
self.info("Starting up server...")
self.trace("** System encoding (in): %s" % sys.stdin.encoding)
self.trace("** System encoding (out): %s" % sys.stdout.encoding)
self.info("Settings, Logger and CLI ready")
DbManager.initialize()
DbManager.instance().isUp()
self.info("Database manager ready")
WebServer.isUp()
self.info("Web server ready")
# Initialize the core
Context.initialize()
Context.instance().setStartTime()
Context.instance().setMysqlVersion()
Context.instance().setApacheVersion()
Context.instance().setPhpVersion()
Context.instance().synchronizeDynamicCfg()
if Settings.getInt('Server', 'use-ifconfig'):
Context.instance().listEths()
else:
Context.instance().listEthsNew()
Context.instance().listRoutes()
self.info("Context ready")
self.deploy()
self.info("Symbolic links created")
ProjectsManager.initialize(context=Context.instance())
self.info("Projects Manager ready")
UsersManager.initialize(context=Context.instance())
self.info("Users Manager ready")
StatsManager.initialize()
self.info("Stats Manager ready")
TaskManager.initialize(statsmgr=StatsManager.instance(),
context=Context)
self.info("Task Manager ready")
# Initialize all repositories
RepoTests.initialize(context=Context.instance(),
taskmgr=TaskManager.instance())
self.info("Repo manager for tests ready")
RepoArchives.initialize(context=Context.instance(),
taskmgr=TaskManager.instance())
self.info("Repo manager for archives ready")
RepoAdapters.initialize(context=Context.instance(),
taskmgr=TaskManager.instance())
StorageDataAdapters.initialize(context=Context.instance())
self.info("Adapters Manager and Storage Data ready")
RepoLibraries.initialize(context=Context.instance(),
taskmgr=TaskManager.instance())
self.info("Libraries adapters manager ready")
RepoPublic.initialize()
self.info("Repo manager for public area is ready")
HelperManager.initialize()
self.info("Helper manager ready")
ProbesManager.initialize(context=Context.instance())
self.info("Probes Manager ready")
AgentsManager.initialize(context=Context.instance())
self.info("Agents Manager ready")
ToolboxManager.initialize()
self.info("Toolbox Manager ready")
# Initialize all interfaces
self.info("Starting ESI on %s:%s" % (Settings.get(
'Bind', 'ip-esi'), Settings.getInt('Bind', 'port-esi')))
EventServerInterface.initialize(
listeningAddress=(Settings.get('Bind', 'ip-esi'),
Settings.getInt('Bind', 'port-esi')),
sslSupport=Settings.getInt('Client_Channel', 'channel-ssl'),
wsSupport=Settings.getInt('Client_Channel',
'channel-websocket-support'),
context=Context.instance())
self.info("Starting TSI on %s:%s" % (Settings.get(
'Bind', 'ip-tsi'), Settings.getInt('Bind', 'port-tsi')))
TestServerInterface.initialize(listeningAddress=(Settings.get(
'Bind', 'ip-tsi'), Settings.getInt('Bind', 'port-tsi')),
statsmgr=StatsManager.instance(),
context=Context.instance())
self.info("Starting RSU on %s:%s" % (Settings.get(
'Bind', 'ip-rsi'), Settings.getInt('Bind', 'port-rsi')))
RestServerInterface.initialize(
listeningAddress=(Settings.get('Bind', 'ip-rsi'),
Settings.getInt('Bind', 'port-rsi')))
self.info("Starting PSI on %s:%s" % (Settings.get(
'Bind', 'ip-psi'), Settings.getInt('Bind', 'port-psi')))
ProbeServerInterface.initialize(
listeningAddress=(Settings.get('Bind', 'ip-psi'),
Settings.getInt('Bind', 'port-psi')),
sslSupport=Settings.getInt('Probe_Channel', 'channel-ssl'),
wsSupport=Settings.getInt('Probe_Channel',
'channel-websocket-support'),
context=Context.instance())
self.info("Starting ASI on %s:%s" % (Settings.get(
'Bind', 'ip-asi'), Settings.getInt('Bind', 'port-asi')))
AgentServerInterface.initialize(
listeningAddress=(Settings.get('Bind', 'ip-asi'),
Settings.getInt('Bind', 'port-asi')),
sslSupport=Settings.getInt('Agent_Channel', 'channel-ssl'),
wsSupport=Settings.getInt('Agent_Channel',
'channel-websocket-support'),
tsi=TestServerInterface.instance(),
context=Context.instance())
# Start on modules
RestServerInterface.instance().start()
self.info("RSI is listening on tcp://%s:%s" % (Settings.get(
'Bind', 'ip-rsi'), Settings.get('Bind', 'port-rsi')))
EventServerInterface.instance().startSA()
self.info("ESI is listening on tcp://%s:%s" % (Settings.get(
'Bind', 'ip-esi'), Settings.get('Bind', 'port-esi')))
TestServerInterface.instance().startSA()
self.info("TSI is listening on tcp://%s:%s" % (Settings.get(
'Bind', 'ip-tsi'), Settings.get('Bind', 'port-tsi')))
ProbeServerInterface.instance().startSA()
self.info("PSI is listening on tcp://%s:%s" % (Settings.get(
'Bind', 'ip-psi'), Settings.get('Bind', 'port-psi')))
AgentServerInterface.instance().startSA()
self.info("ASI is listening on tcp://%s:%s" % (Settings.get(
'Bind', 'ip-asi'), Settings.get('Bind', 'port-asi')))
# Now start the scheduler and reload tasks
taskReloaded = TaskManager.instance().loadBackups()
if taskReloaded is None:
self.info("Reload tasks disabled")
elif taskReloaded:
self.info("Tasks reloaded")
else:
self.error("Failed to reload tasks")
self.info("Schedule automatic backups...")
if Settings.getInt('Backups', 'tests'):
RepoAdapters.instance().scheduleBackup()
self.info("Backup tests scheduled")
else:
self.info("Backup tests disabled")
if Settings.getInt('Backups', 'adapters'):
RepoTests.instance().scheduleBackup()
self.info("Backup adapters scheduled")
else:
self.info("Backup adapters disabled")
if Settings.getInt('Backups', 'libraries'):
RepoLibraries.instance().scheduleBackup()
self.info("Backup libraries scheduled")
else:
self.info("Backup libraries disabled")
if Settings.getInt('Backups', 'archives'):
RepoArchives.instance().scheduleBackup()
self.info("Backup archives scheduled")
else:
self.info("Backup archives disabled")
except Exception as e:
self.error("Unable to start server: %s" % str(e))
self.cleanup()
sys.exit(3)
stoptime = time.time()
self.info("%s successfully started (in %s sec.)" %
(Settings.get('Server', 'name'), int(stoptime - starttime)))
self.setrunning()
self.run()
def zipData(self, dirToday, dirTest, destPathZip, replayId, projectId=0, virtualName = ""):
"""
Zip data by adapters and notify users
@param dirToday: date YYYY-MM-DD
@type dirToday: string
@param dirTest:
@type dirTest: string
@param destPathZip:
@type destPathZip: string
@param replayId:
@type replayId: string
"""
ret = False
try:
mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday)
testDir = "%s/%s" % (mainDir, dirTest)
for x in os.listdir(testDir):
# is directory
if not os.path.isfile( '%s/%s' % (testDir,x) ):
pathTozip = '%s/%s' % (testDir,x)
# empty folders are not zipped
if len( os.listdir(pathTozip) ) == 0:
continue
# prepare the file name
tp = time.strftime( "%Y-%m-%d_%H-%M-%S", time.localtime(time.time()) ) \
+ ".%3.3d" % int((time.time() * 1000) % 1000 )
fileName = "%s_%s_%s_%s" % (self.prefixAdapters, x, tp , replayId)
# zip the folder
zipped = self.toZip(
file=pathTozip,
filename="%s/%s.zip" % (destPathZip, fileName),
extToInclude = [], # include all files
keepTree=False
)
if zipped == self.context.CODE_OK:
# notify users
if Settings.getInt( 'Notifications', 'archives'):
size_ = os.path.getsize( "%s/%s.zip" % (destPathZip, fileName) )
notif = {}
m = [ { "type": "folder", "name": dirToday, "project": "%s" % projectId,
"content": [ { "type": "folder", "name": dirTest, "project": "%s" % projectId, "virtual-name": virtualName, "content": [ {"type": "file", "name": "%s.zip" % fileName, 'size': str(size_), "project": "%s" % projectId } ]} ] } ]
notif['archive'] = m
data = ( 'archive', ( None, notif) )
ESI.instance().notifyByUserTypes(body = data,
admin=True,
monitor=False,
tester=True)
else:
self.error( 'error to zip data adapters' )
ret = False
break
ret = True
except Exception as e:
self.error( 'unable to zip data: %s' % str(e) )
return ret