def __init__(self, listeningAddress, agentName='ESI', sslSupport=False, wsSupport=False, context=None): """ Event server interface @param listeningAddress: @type listeningAddress: @param agentName: agent name used on request @type agentName: string """ NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), sslSupport=sslSupport, wsSupport=wsSupport, certFile='%s/%s' % (Settings.getDirExec(), Settings.get('Client_Channel', 'channel-ssl-cert')), keyFile='%s/%s' % (Settings.getDirExec(), Settings.get('Client_Channel', 'channel-ssl-key')), pickleVer=Settings.getInt('Network', 'pickle-version')) self.__mutex__ = threading.RLock() self.context = context
def __init__(self, listeningAddress, agentName='TSI', statsmgr=None, context=None): """ Constructs TCP Server Inferface @param listeningAddress: @type listeningAddress: """ NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), pickleVer=Settings.getInt('Network', 'pickle-version')) self.statsmgr = statsmgr self.context = context self.__mutex__ = threading.RLock() self.__fifoThread = None self.tests = { } # { 'task-id': Boolean} # test register, with background running or not self.testsConnected = {} # all tests connected
def __init__ (self, listeningAddress, agentName = 'PSI', sslSupport=False, wsSupport=False, context=None): """ Construct Probe Server Interface @param listeningAddress: @type listeningAddress: @param agentName: @type agentName: string """ NetLayerLib.ServerAgent.__init__(self, listeningAddress = listeningAddress, agentName = agentName, keepAliveInterval= Settings.getInt('Network', 'keepalive-interval' ), inactivityTimeout=Settings.getInt( 'Network', 'inactivity-timeout' ), responseTimeout=Settings.getInt( 'Network', 'response-timeout' ), selectTimeout=Settings.get( 'Network', 'select-timeout' ), sslSupport=sslSupport, wsSupport=wsSupport, certFile='%s/%s' % (Settings.getDirExec(), Settings.get( 'Probe_Channel', 'channel-ssl-cert' )), keyFile='%s/%s' % (Settings.getDirExec(), Settings.get( 'Probe_Channel', 'channel-ssl-key' )), pickleVer=Settings.getInt( 'Network', 'pickle-version' ) ) self.context = context self.__mutex = threading.RLock() self.probesRegistered = {} self.probesPublicIp = {}
def createBackup(self, backupName): """ Create a backup of all libraries @type backupName: @param backupName: @return: @rtype: """ ret = self.context.CODE_ERROR try: backupIndex = self.getLastBackupIndex( pathBackups=self.destBackup ) backupDate = self.getTimestamp() backupFilename = '%s%s_%s_%s' % ( self.prefixBackup, backupIndex, backupName, backupDate ) # new in v14.0.0: create tar gz if Settings.getInt( 'Backups', 'libraries-dest-tar-gz' ): self.trace( "backup libraries to %s/%s.tar.gz" % (self.destBackup,backupFilename) ) DEVNULL = open(os.devnull, 'w') __cmd__ = "%s cvfz %s/%s.tar.gz -C %s ." % (Settings.get( 'Bin', 'tar' ), self.destBackup, backupFilename, self.testsPath) ret = subprocess.call(__cmd__, shell=True, stdout=DEVNULL, stderr=DEVNULL) if ret: raise Exception("unable to tar sut libraries pkg") ret = self.context.CODE_OK # create a zip file if Settings.getInt( 'Backups', 'libraries-dest-zip' ): self.trace( "backup libraries to %s/%s.zip" % (self.destBackup,backupFilename) ) zipped = self.zipFolder(folderPath=self.testsPath, zipName="%s.zip" % backupFilename, zipPath=self.destBackup, ignoreExt=['.pyc', '.pyo']) ret = zipped if zipped == self.context.CODE_OK: self.info( "backup libraries successfull: %s" % backupFilename ) # now notify all connected admin users backupSize = os.path.getsize( "%s/%s.zip" % (self.destBackup, backupFilename) ) notif = {} notif['repo-libraries'] = {} notif['repo-libraries']['backup'] = {'name': backupName, 'date': backupDate, 'size': backupSize, 'fullname': "%s.zip" % backupFilename } data = ( 'repositories', ( None, notif) ) ESI.instance().notifyAllAdmins(body = data) else: self.error( "backup libraries %s failed" % backupFilename ) except Exception as e: raise Exception( "[createBackup] %s" % str(e) ) return ret
def updateScreen(self, filename, xmlPath, x=0, y=0, w=0, h=0, reloadMode=False): """ Update the screen """ self.imagePath = filename if not reloadMode: self.tableModel.mylist = [] self.tableModel.beginResetModel() self.tableModel.endResetModel() pixmap = QPixmap(filename) if pixmap is not None: self.origWidth = pixmap.width() self.origHeight = pixmap.height() self.screenResolutionLabel.setText( "Resolution=%sx%s" % (self.origWidth, self.origHeight)) #portrait if self.origWidth < self.origHeight: pixmap = pixmap.scaledToHeight(Settings.getInt( 'MobileAndroid', 'resolution-screen-height'), mode=Qt.SmoothTransformation) self.mobileImageLabel.setPixmap(pixmap) else: pixmap = pixmap.scaledToWidth(Settings.getInt( 'MobileAndroid', 'resolution-screen-width'), mode=Qt.SmoothTransformation) self.mobileImageLabel.setPixmap(pixmap) self.drawRectangle(x=x, y=y, w=w, h=h) self.resize(pixmap.width(), pixmap.height()) # convert xml to dict if len(xmlPath): f = QFile(xmlPath) if f.open(QIODevice.ReadOnly): document = QDomDocument() if document.setContent(f): newModel = DomModel(document, self) self.mobileTreeView.setModel(newModel) self.mobileTreeView.expandAll() self.mobileTreeView.resizeColumnToContents(0) f.close()
def isUp(): """ Check if the web server if ready (apache) """ try: # init the http timeoutVal = Settings.getInt('Boot', 'timeout-http-server') http = httplib2.Http(timeout=timeoutVal, disable_ssl_certificate_validation=True, ca_certs="%s/Libs/cacerts.txt" % Settings.getDirExec()) http.add_credentials(Settings.get('Web', 'login'), Settings.get('Web', 'password')) http.force_exception_to_status_code = True scheme = 'http' portHttp = Settings.get('Web', 'http-port') if Settings.getInt('Web', 'https'): scheme = 'https' portHttp = Settings.get('Web', 'https-port') uri = '%s://%s:%s/%s/index.php' % (scheme, Settings.get( 'Web', 'fqdn'), portHttp, Settings.get('Web', 'path')) timeout = False go = False startTime = time.time() while (not go) and (not timeout): # timeout elapsed ? if (time.time() - startTime) >= timeoutVal: timeout = True else: # test the web server Logger.debug("Get index: %s" % uri) resp, content = http.request(uri, "GET") if resp['status'] == '200': Logger.debug("200 OK received") go = True else: Logger.debug("response incorrect (%s)" % resp['status']) Logger.debug("response (%s)" % resp) Logger.debug("response content (%s)" % content) Logger.debug("retry in %s second" % Settings.get('Web', 'retry-connect')) time.sleep(int(Settings.get('Web', 'retry-connect'))) if timeout: raise Exception("timeout") except Exception as e: raise Exception("server web not ready: %s" % str(e))
def startDefault(self): """ Start all default probes @return: @rtype: """ started = True self.trace("start default tools") if self.TOOLS_INSTALLED: if not Settings.getInt('Boot', 'start-local-tools'): self.trace("auto start disabled") started = None else: self.info("Starting default local tools") for p in ProbesManager.instance().getDefaultProbes(): if bool(eval(p['enable'])): ret = ProbesManager.instance().startProbe( ptype=p['type'], pname=p['name'], pdescr=p['description'], pdefault=True) if ret != 0: started = False for a in AgentsManager.instance().getDefaultAgents(): if bool(eval(a['enable'])): ret = AgentsManager.instance().startAgent( atype=a['type'], aname=a['name'], adescr=a['description'], adefault=True) if ret != 0: started = False else: started = None return started
def getNbTests(self): """ Get number of tests """ nb = [ {'nbta': 0, 'nbtc': 0, 'nbsc': 0, 'nbtg': 0, 'nbtp': 0, 'nbts': 0, 'nbtu': 0} ] if not Settings.getInt( 'MySql', 'read-test-statistics'): return nb self.trace('get number of tests') ok, nb_entries = DbManager.instance().querySQL( columnName=True, query="""SELECT count(*) as nbsc, (SELECT count(*) FROM `%s`) as nbtg, (SELECT count(*) FROM `%s`) as nbtp, (SELECT count(*) FROM `%s`) as nbts, (SELECT count(*) FROM `%s`) as nbtu, (SELECT count(*) FROM `%s`) as nbta, (SELECT count(*) FROM `%s`) as nbtc FROM `%s`;""" % ( self.dbt_testglobals, self.dbt_testplans, self.dbt_testsuites, self.dbt_testunits, self.dbt_testabstracts, self.dbt_testcases, self.dbt_scripts ) ) if ok: self.trace('number of tests successfully retrieved') return nb_entries else: self.error('unable to retrieve the number of tests') return nb
def __init__(self): """ Construct toolbox Manager """ self.pkgsToolsPath = "%s/%s/%s/linux2/" % ( Settings.getDirExec(), Settings.get( 'Paths', 'packages'), Settings.get('Paths', 'tools')) self.info('Detecting local tools to deploy...') pkg = self.preInstall() if Settings.getInt('WebServices', 'local-tools-enabled'): if pkg is not None: self.info('Deploying local tools %s...' % pkg) self.installPkgV2(pkgName=pkg) self.TOOLS_INSTALLED = False try: import Toolbox self.TOOLS_INSTALLED = True self.info("Local tools are installed") except Exception as e: self.info("Local tools are NOT installed") self.trace("More details: %s" % unicode(e).encode('utf-8')) self.configsFile = None self.__pids__ = {}
def addResultTestSuite(self, tsResult, fromUser, tsDuration, nbTc, prjId): """ Add testcase result to db @param tsResult: PASS, FAIL, UNDEFINED @type tsResult: str @param fromUser: username @type fromUser: str @param tsDuration: @type tsDuration: str @param nbTc: @type nbTc: str """ if not Settings.getInt('MySql', 'insert-test-statistics'): return self.trace('add result ts %s' % tsResult) self.__mutex__.acquire() try: strDate = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) # ret, rows = DbManager.instance().querySQL( query= "INSERT INTO `%s` (date, result, user_id, duration, nbtc, project_id) VALUES ('%s','%s',%s,'%s',%s,%s)" % (self.dbt_testsuites, strDate, tsResult, fromUser, tsDuration, nbTc, prjId)) if not ret: raise Exception("unable to add result testsuite in db") except Exception as e: self.error(e) self.__mutex__.release()
def addResultScript (self, scriptResult, scriptUser, scriptDuration, scriptProject ): """ Add script result to db @param scriptResult: COMPLETE, ERROR, KILLED, etc... @type scriptResult: str @param scriptUser: username @type scriptUser: str @param scriptDuration: duration @type scriptDuration: str """ if not Settings.getInt( 'MySql', 'insert-test-statistics'): return self.trace('add result sc %s' % scriptResult) try: strDate = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) # ret, rows = DbManager.instance().querySQL( query = "INSERT INTO `%s` (date, result, user_id, duration, project_id) VALUES ('%s','%s',%s,'%s','%s')" % (self.dbt_scripts, strDate, scriptResult, scriptUser, scriptDuration, scriptProject ) ) if not ret: raise Exception("failed to add result script in db") except Exception as e: self.error( e ) else: if self.notifyUsers: data = ( 'stats', ( None, self.getStats() ) ) ESI.instance().notifyByUserTypes( body=data, admin=True, leader=True, tester=False, developer=False)
def stopDefault(self): """ Stop all default probes @return: @rtype: """ stopped = True self.trace("Stop all default tools") if PSI.instance() is None: self.trace("psi not ready then nothing todo") return None if self.TOOLS_INSTALLED: if not Settings.getInt('Boot', 'start-probes'): self.trace( "The auto start option is disabled then there are no default probes started" ) stopped = None else: self.info("Stopping default tools") for p in ProbesManager.instance().getDefaultProbes(): if bool(eval(p['enable'])): ret = ProbesManager.instance().stopProbe( pname=p['name']) if not ret: stopped = False for a in AgentsManager.instance().getDefaultProbes(): if bool(eval(a['enable'])): ret = AgentsManager.instance().stopAgent( aname=a['name']) if not ret: stopped = False else: stopped = None return stopped
def onRegistration (self, client, tid, request): """ Called on the registration of a new agents @param client: @type client: @param tid: @type tid: @param request: @type request: """ self.trace("on registration" ) self.__mutex.acquire() doNotify=False if request['userid'] in self.agentsRegistered: self.info('duplicate agents registration: %s' % request['userid'] ) NetLayerLib.ServerAgent.failed(self, client, tid) else: if not ('type' in request['body']): self.error('type missing in request: %s' % request['body'] ) NetLayerLib.ServerAgent.failed(self, client, tid) else: if request['body']['type'] != ClientAgent.TYPE_AGENT_AGENT: self.error('agent type refused: %s' % request['body']['type']) NetLayerLib.ServerAgent.forbidden(self, client, tid) else: tpl = { 'address' : client, 'version': request['body']['version'], 'description': request['body']['description']['details'], 'auto-startup': request['body']['description']['default'], 'type': request['body']['name'], 'start-at': request['body']['start-at'], 'publicip': self.agentsPublicIp[client] } if not Settings.getInt( 'WebServices', 'remote-agents-enabled' ): if not tpl['auto-startup']: self.info('remote agents registration not authorized') NetLayerLib.ServerAgent.forbidden(self, client, tid) else: self.agentsRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.trace( 'Local agent registered: Name="%s"' % request['userid'] ) doNotify = True else: self.agentsRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.info( 'Remote agent registered: Name="%s"' % request['userid'] ) doNotify = True if doNotify: # Notify all connected users notif = ( 'agents', ( 'add', self.getAgents() ) ) ESI.instance().notifyByUserTypes(body = notif, admin=True, monitor=False, tester=True) self.__mutex.release()
def moveNewFile (self, data): """ Move the file from the temp unix to the test result storage Deprecated function thanks to the migration to python3 on probe side @type data: @param data: """ self.trace(" moving file" ) try: testResult = data['result-path'] fileName = data['filename'] self.trace( 'move %s to %s' % ( fileName, testResult ) ) # move file testsResultPath = '%s%s' % ( Settings.getDirExec(),Settings.get( 'Paths', 'testsresults' ) ) shutil.copyfile( src = '/tmp/%s' % fileName, dst = '%s/%s/%s' % (testsResultPath, testResult, fileName) ) except Exception as e: self.error( "unable to move the new file: %s" % e ) else: try: # now notify all connected users size_ = os.path.getsize( '%s/%s/%s' % (testsResultPath, testResult, fileName) ) if testResult.startswith('/'): testResult = testResult[1:] tmp = testResult.split('/', 1) projectId = tmp[0] tmp = tmp[1].split('/', 1) mainPathTozip= tmp[0] subPathTozip = tmp[1] if Settings.getInt( 'Notifications', 'archives'): m = [ { "type": "folder", "name": mainPathTozip, "project": "%s" % projectId, "content": [ { "type": "folder", "name": subPathTozip, "project": "%s" % projectId, "content": [ { "project": "%s" % projectId, "type": "file", "name": fileName, 'size': str(size_) } ]} ] } ] notif = {} notif['archive'] = m notif['stats-repo-archives'] = { 'nb-zip':1, 'nb-trx':0, 'nb-tot': 1, 'mb-used': RepoArchives.instance().getSizeRepo(folder=RepoArchives.instance().testsPath), 'mb-free': RepoArchives.instance().freeSpace(p=RepoArchives.instance().testsPath) } data = ( 'archive', ( None, notif) ) ESI.instance().notifyByUserAndProject(body = data, admin=True, monitor=False, tester=True, projectId="%s" % projectId) except Exception as e: self.error( "unable to notify users for this new file: %s" % e ) # clean temp dir try: os.remove( '/tmp/%s' % fileName ) except Exception as e: pass
def zipDataV2(self, dirToday, dirTest, destPathZip, replayId, projectId=0, virtualName = ""): """ Zip data by adapters and notify users in just one zip @param dirToday: date YYYY-MM-DD @type dirToday: string @param dirTest: @type dirTest: string @param destPathZip: @type destPathZip: string @param replayId: @type replayId: string """ self.trace("Starting to zip all adapters logs") ret = False try: mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday) testDir = "%s/%s" % (mainDir, dirTest) # prepare the file name tp = time.strftime( "%Y-%m-%d_%H-%M-%S", time.localtime(time.time()) ) \ + ".%3.3d" % int((time.time() * 1000) % 1000 ) fileName = "%s_%s_%s" % (self.prefixAdaptersAll, tp , replayId) # zip the folder zipped = self.zipFolder(folderPath=testDir, zipName="%s.zip" % fileName, zipPath=destPathZip) if zipped == self.context.CODE_OK: # notify users if Settings.getInt( 'Notifications', 'archives'): size_ = os.path.getsize( "%s/%s.zip" % (destPathZip, fileName) ) notif = {} m = [ { "type": "folder", "name": dirToday, "project": "%s" % projectId, "content": [ { "type": "folder", "name": dirTest, "project": "%s" % projectId, "virtual-name": virtualName, "content": [ {"type": "file", "name": "%s.zip" % fileName, 'size': str(size_), "project": "%s" % projectId } ]} ] } ] notif['archive'] = m data = ( 'archive', ( None, notif) ) ESI.instance().notifyByUserTypes(body = data, admin=True, monitor=False, tester=True) ret = True else: self.error( 'error to zip data adapters' ) ret = False except Exception as e: self.error( 'unable to zip data adapters v2: %s' % str(e) ) return ret
def __init__ (self): """ Statistics Manager for tests """ self.__mutex__ = threading.RLock() self.dbt_testcases = '%s-testcases-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_testunits = '%s-testunits-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_testabstracts = '%s-testabstracts-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_testsuites = '%s-testsuites-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_testplans = '%s-testplans-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_testglobals = '%s-testglobals-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_scripts = '%s-scripts-stats' % Settings.get( 'MySql', 'table-prefix') self.dbt_writing = '%s-writing-stats' % Settings.get( 'MySql', 'table-prefix') self.notifyUsers = Settings.getInt( 'Notifications', 'statistics')
def getNbOfTestsuites(self): """ Return the number of testsuites, used on the module context @return: nb ts in database @rtype: int """ nb = 0 if not Settings.getInt( 'MySql', 'read-test-statistics'): return nb self.trace('get number of ts') ok, nb_entries = DbManager.instance().querySQL( query="SELECT count(*) FROM `%s`" % self.dbt_testsuites ) if ok: nb = int( nb_entries[0][0] ) self.trace('number of ts: %s' % nb) return nb
def isUp(self): """ Try to connect to the database Detect the version of the mysql server """ timeoutVal = Settings.getInt('Boot', 'timeout-sql-server') timeout = False go = False startTime = time.time() while (not go) and (not timeout): # timeout elapsed ? if (time.time() - startTime) >= timeoutVal: timeout = True else: try: self.trace("try to connect to the database") conn = MySQLdb.connect(host=Settings.get('MySql', 'ip'), user=Settings.get('MySql', 'user'), passwd=Settings.get('MySql', 'pwd'), db=Settings.get('MySql', 'db'), unix_socket=Settings.get( 'MySql', 'sock')) go = True self.trace("connection successful") except MySQLdb.Error as e: Logger.debug("connect to the database failed: %s" % str(e)) Logger.debug("retry in %s second" % Settings.get('MySql', 'retry-connect')) time.sleep(int(Settings.get('MySql', 'retry-connect'))) if timeout: raise Exception("db manager not ready: timeout") if go: self.trace("retrieve mysql version") cursor = conn.cursor() cursor.execute("SELECT VERSION()") row = cursor.fetchone() cursor.close() conn.close() self.dbVersion = row[0] self.trace(self.dbVersion)
def getTree(self, b64=False, fullTree=False, project=1): """ Return tree of files """ nb = Settings.getInt('WebServices', 'nb-archives') if nb == -1: nb = None if nb == 0: return (0, 0, [], {}) if fullTree: nb = None success = os.path.exists("%s/%s" % (self.testsPath, project)) if not success: return (0, 0, [], {}) else: return self.getListingFilesV2(path="%s/%s" % (self.testsPath, project), nbDirs=nb, project=project, archiveMode=True)
def createResultLog(self, testsPath, logPath, logName, logData): """ Create result log """ self.trace("create result log=%s to %s" % (logName, logPath)) try: # write the file f = open("%s/%s/%s" % (testsPath, logPath, logName), 'wb') f.write(base64.b64decode(logData)) f.close() # notify all users size_ = os.path.getsize("%s/%s/%s" % (testsPath, logPath, logName)) # extract project id if logPath.startswith('/'): logPath = logPath[1:] tmp = logPath.split('/', 1) projectId = tmp[0] tmp = tmp[1].split('/', 1) mainPathTozip = tmp[0] subPathTozip = tmp[1] if Settings.getInt('Notifications', 'archives'): m = [{ "type": "folder", "name": mainPathTozip, "project": "%s" % projectId, "content": [{ "type": "folder", "name": subPathTozip, "project": "%s" % projectId, "content": [{ "project": "%s" % projectId, "type": "file", "name": logName, 'size': str(size_) }] }] }] notif = {} notif['archive'] = m notif['stats-repo-archives'] = { 'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1, 'mb-used': self.getSizeRepoV2(folder=self.testsPath), 'mb-free': self.freeSpace(p=self.testsPath) } data = ('archive', (None, notif)) ESI.instance().notifyByUserAndProject(body=data, admin=True, leader=False, tester=True, developer=False, projectId="%s" % projectId) except Exception as e: self.error("unable to create result log: %s" % e) return False return True
self.error('type missing in request: %s' % request['body'] ) NetLayerLib.ServerAgent.failed(self, client, tid) else: if request['body']['type'] != ClientAgent.TYPE_AGENT_AGENT: self.error('agent type refused: %s' % request['body']['type']) NetLayerLib.ServerAgent.forbidden(self, client, tid) else: tpl = { 'address' : client, 'version': request['body']['version'], 'description': request['body']['description']['details'], 'auto-startup': request['body']['description']['default'], 'type': request['body']['name'], 'start-at': request['body']['start-at'], 'publicip': self.agentsPublicIp[client] } if not Settings.getInt( 'WebServices', 'remote-agents-enabled' ): if not tpl['auto-startup']: self.info('remote agents registration not authorized') NetLayerLib.ServerAgent.forbidden(self, client, tid) else: self.agentsRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.trace( 'Local agent registered: Name="%s"' % request['userid'] ) doNotify = True else: self.agentsRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.info( 'Remote agent registered: Name="%s"' % request['userid'] ) doNotify = True if doNotify:
def initialize(self): """ Starts all modules Exit if the service is alreayd running or if the config file is missing """ starttime = time.time() if self.isrunning(): sys.stdout.write(" (server is already running)") sys.exit(1) self.daemonize() try: # Initialize self.info("Starting up server...") self.trace("** System encoding (in): %s" % sys.stdin.encoding) self.trace("** System encoding (out): %s" % sys.stdout.encoding) self.info("Settings, Logger and CLI ready") DbManager.initialize() DbManager.instance().isUp() self.info("Database manager ready") WebServer.isUp() self.info("Web server ready") # Initialize the core Context.initialize() Context.instance().setStartTime() Context.instance().setMysqlVersion() Context.instance().setApacheVersion() Context.instance().setPhpVersion() Context.instance().synchronizeDynamicCfg() if Settings.getInt('Server', 'use-ifconfig'): Context.instance().listEths() else: Context.instance().listEthsNew() Context.instance().listRoutes() self.info("Context ready") self.deploy() self.info("Symbolic links created") ProjectsManager.initialize(context=Context.instance()) self.info("Projects Manager ready") UsersManager.initialize(context=Context.instance()) self.info("Users Manager ready") StatsManager.initialize() self.info("Stats Manager ready") TaskManager.initialize(statsmgr=StatsManager.instance(), context=Context) self.info("Task Manager ready") # Initialize all repositories RepoTests.initialize(context=Context.instance(), taskmgr=TaskManager.instance()) self.info("Repo manager for tests ready") RepoArchives.initialize(context=Context.instance(), taskmgr=TaskManager.instance()) self.info("Repo manager for archives ready") RepoAdapters.initialize(context=Context.instance(), taskmgr=TaskManager.instance()) StorageDataAdapters.initialize(context=Context.instance()) self.info("Adapters Manager and Storage Data ready") RepoLibraries.initialize(context=Context.instance(), taskmgr=TaskManager.instance()) self.info("Libraries adapters manager ready") RepoPublic.initialize() self.info("Repo manager for public area is ready") HelperManager.initialize() self.info("Helper manager ready") ProbesManager.initialize(context=Context.instance()) self.info("Probes Manager ready") AgentsManager.initialize(context=Context.instance()) self.info("Agents Manager ready") ToolboxManager.initialize() self.info("Toolbox Manager ready") # Initialize all interfaces self.info("Starting ESI on %s:%s" % (Settings.get( 'Bind', 'ip-esi'), Settings.getInt('Bind', 'port-esi'))) EventServerInterface.initialize( listeningAddress=(Settings.get('Bind', 'ip-esi'), Settings.getInt('Bind', 'port-esi')), sslSupport=Settings.getInt('Client_Channel', 'channel-ssl'), wsSupport=Settings.getInt('Client_Channel', 'channel-websocket-support'), context=Context.instance()) self.info("Starting TSI on %s:%s" % (Settings.get( 'Bind', 'ip-tsi'), Settings.getInt('Bind', 'port-tsi'))) TestServerInterface.initialize(listeningAddress=(Settings.get( 'Bind', 'ip-tsi'), Settings.getInt('Bind', 'port-tsi')), statsmgr=StatsManager.instance(), context=Context.instance()) self.info("Starting RSU on %s:%s" % (Settings.get( 'Bind', 'ip-rsi'), Settings.getInt('Bind', 'port-rsi'))) RestServerInterface.initialize( listeningAddress=(Settings.get('Bind', 'ip-rsi'), Settings.getInt('Bind', 'port-rsi'))) self.info("Starting PSI on %s:%s" % (Settings.get( 'Bind', 'ip-psi'), Settings.getInt('Bind', 'port-psi'))) ProbeServerInterface.initialize( listeningAddress=(Settings.get('Bind', 'ip-psi'), Settings.getInt('Bind', 'port-psi')), sslSupport=Settings.getInt('Probe_Channel', 'channel-ssl'), wsSupport=Settings.getInt('Probe_Channel', 'channel-websocket-support'), context=Context.instance()) self.info("Starting ASI on %s:%s" % (Settings.get( 'Bind', 'ip-asi'), Settings.getInt('Bind', 'port-asi'))) AgentServerInterface.initialize( listeningAddress=(Settings.get('Bind', 'ip-asi'), Settings.getInt('Bind', 'port-asi')), sslSupport=Settings.getInt('Agent_Channel', 'channel-ssl'), wsSupport=Settings.getInt('Agent_Channel', 'channel-websocket-support'), tsi=TestServerInterface.instance(), context=Context.instance()) # Start on modules RestServerInterface.instance().start() self.info("RSI is listening on tcp://%s:%s" % (Settings.get( 'Bind', 'ip-rsi'), Settings.get('Bind', 'port-rsi'))) EventServerInterface.instance().startSA() self.info("ESI is listening on tcp://%s:%s" % (Settings.get( 'Bind', 'ip-esi'), Settings.get('Bind', 'port-esi'))) TestServerInterface.instance().startSA() self.info("TSI is listening on tcp://%s:%s" % (Settings.get( 'Bind', 'ip-tsi'), Settings.get('Bind', 'port-tsi'))) ProbeServerInterface.instance().startSA() self.info("PSI is listening on tcp://%s:%s" % (Settings.get( 'Bind', 'ip-psi'), Settings.get('Bind', 'port-psi'))) AgentServerInterface.instance().startSA() self.info("ASI is listening on tcp://%s:%s" % (Settings.get( 'Bind', 'ip-asi'), Settings.get('Bind', 'port-asi'))) # Now start the scheduler and reload tasks taskReloaded = TaskManager.instance().loadBackups() if taskReloaded is None: self.info("Reload tasks disabled") elif taskReloaded: self.info("Tasks reloaded") else: self.error("Failed to reload tasks") self.info("Schedule automatic backups...") if Settings.getInt('Backups', 'tests'): RepoAdapters.instance().scheduleBackup() self.info("Backup tests scheduled") else: self.info("Backup tests disabled") if Settings.getInt('Backups', 'adapters'): RepoTests.instance().scheduleBackup() self.info("Backup adapters scheduled") else: self.info("Backup adapters disabled") if Settings.getInt('Backups', 'libraries'): RepoLibraries.instance().scheduleBackup() self.info("Backup libraries scheduled") else: self.info("Backup libraries disabled") if Settings.getInt('Backups', 'archives'): RepoArchives.instance().scheduleBackup() self.info("Backup archives scheduled") else: self.info("Backup archives disabled") except Exception as e: self.error("Unable to start server: %s" % str(e)) self.cleanup() sys.exit(3) stoptime = time.time() self.info("%s successfully started (in %s sec.)" % (Settings.get('Server', 'name'), int(stoptime - starttime))) self.setrunning() self.run()
self.error('type missing in request: %s' % request['body'] ) NetLayerLib.ServerAgent.failed(self, client, tid) else: if request['body']['type'] != ClientAgent.TYPE_AGENT_PROBE: self.error('probe type refused: %s' % request['body']['type']) NetLayerLib.ServerAgent.forbidden(self, client, tid) else: tpl = { 'address' : client, 'version': request['body']['version'], 'description': request['body']['description']['details'], 'auto-startup': request['body']['description']['default'], 'type': request['body']['name'], 'start-at': request['body']['start-at'], 'publicip': self.probesPublicIp[client] } if not Settings.getInt( 'WebServices', 'remote-probes-enabled' ): if not tpl['auto-startup']: self.info('remote probes registration not authorized') NetLayerLib.ServerAgent.forbidden(self, client, tid) else: self.probesRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.trace( 'Local probe registered: Name="%s"' % request['userid'] ) doNotify = True else: self.probesRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.info( 'Remote probe registered: Name="%s"' % request['userid'] ) doNotify = True if doNotify: # Notify all connected users
def zipData(self, dirToday, dirTest, destPathZip, replayId, projectId=0, virtualName = ""): """ Zip data by adapters and notify users @param dirToday: date YYYY-MM-DD @type dirToday: string @param dirTest: @type dirTest: string @param destPathZip: @type destPathZip: string @param replayId: @type replayId: string """ ret = False try: mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday) testDir = "%s/%s" % (mainDir, dirTest) for x in os.listdir(testDir): # is directory if not os.path.isfile( '%s/%s' % (testDir,x) ): pathTozip = '%s/%s' % (testDir,x) # empty folders are not zipped if len( os.listdir(pathTozip) ) == 0: continue # prepare the file name tp = time.strftime( "%Y-%m-%d_%H-%M-%S", time.localtime(time.time()) ) \ + ".%3.3d" % int((time.time() * 1000) % 1000 ) fileName = "%s_%s_%s_%s" % (self.prefixAdapters, x, tp , replayId) # zip the folder zipped = self.toZip( file=pathTozip, filename="%s/%s.zip" % (destPathZip, fileName), extToInclude = [], # include all files keepTree=False ) if zipped == self.context.CODE_OK: # notify users if Settings.getInt( 'Notifications', 'archives'): size_ = os.path.getsize( "%s/%s.zip" % (destPathZip, fileName) ) notif = {} m = [ { "type": "folder", "name": dirToday, "project": "%s" % projectId, "content": [ { "type": "folder", "name": dirTest, "project": "%s" % projectId, "virtual-name": virtualName, "content": [ {"type": "file", "name": "%s.zip" % fileName, 'size': str(size_), "project": "%s" % projectId } ]} ] } ] notif['archive'] = m data = ( 'archive', ( None, notif) ) ESI.instance().notifyByUserTypes(body = data, admin=True, leader=False, tester=True, developer=False) else: self.error( 'error to zip data adapters' ) ret = False break ret = True except Exception as e: self.error( 'unable to zip data: %s' % str(e) ) return ret
def createZip(self, trPath, projectId=1): """ Create a zip @type mainPathTozip: @param mainPathTozip: @type subPathTozip: @param subPathTozip: @return: response code @rtype: int """ ret = self.context.CODE_ERROR mainPathTozip, subPathTozip = trPath.split("/", 1) try: timeArch, milliArch, testName, testUser = subPathTozip.split(".") fulltestNameDecoded = base64.b64decode(testName) logIndex = self.getLastLogIndex(pathZip=trPath, projectId=projectId) tmp = fulltestNameDecoded.rsplit('/', 1) if len(tmp) == 2: testNameDecoded = tmp[1] else: testNameDecoded = tmp[0] fileName = 'logs%s_%s_%s_0' % (str(logIndex), testNameDecoded, self.getTimestamp()) self.trace("zip %s to %s.zip" % (trPath, fileName)) zipped = self.toZip(file="%s/%s/%s" % (self.testsPath, projectId, trPath), filename="%s/%s/%s/%s.zip" % (self.testsPath, projectId, trPath, fileName), fileToExclude=['%s.zip' % fileName], keepTree=False) ret = zipped if zipped == self.context.CODE_OK: self.info("zip successfull: %s" % fileName) if Settings.getInt('Notifications', 'archives'): # now notify all connected users size_ = os.path.getsize( "%s/%s/%s/%s.zip" % (self.testsPath, projectId, trPath, fileName)) m = [{ "type": "folder", "name": mainPathTozip, "project": "%s" % projectId, "content": [{ "type": "folder", "name": subPathTozip, "project": "%s" % projectId, "content": [{ "project": "%s" % projectId, "type": "file", "name": '%s.zip' % fileName, 'size': str(size_) }] }] }] notif = {} notif['archive'] = m notif['stats-repo-archives'] = { 'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1, 'mb-used': self.getSizeRepoV2(folder=self.testsPath), 'mb-free': self.freeSpace(p=self.testsPath) } data = ('archive', (None, notif)) ESI.instance().notifyByUserTypes(body=data, admin=True, leader=False, tester=True, developer=False) else: self.error("zip %s failed" % trPath) except Exception as e: raise Exception("[createZip] %s" % str(e)) return ret
def getStats(self): """ Returns statistics """ ret = {} if not Settings.getInt('MySql', 'read-test-statistics'): nbScriptCompleted = 0 nbScriptError = 0 nbScriptKilled = 0 ret['scripts'] = { 'nb-completed': nbScriptCompleted, 'nb-error': nbScriptError, 'nb-killed': nbScriptKilled } nbTgPass = 0 nbTgFail = 0 nbTgUndef = 0 ret['testglobals'] = { 'nb-pass': nbTgPass, 'nb-fail': nbTgFail, 'nb-undef': nbTgUndef } nbTpPass = 0 nbTpFail = 0 nbTpUndef = 0 ret['testplans'] = { 'nb-pass': nbTpPass, 'nb-fail': nbTpFail, 'nb-undef': nbTpUndef } nbTsPass = 0 nbTsFail = 0 nbTsUndef = 0 ret['testsuites'] = { 'nb-pass': nbTsPass, 'nb-fail': nbTsFail, 'nb-undef': nbTsUndef } nbTuPass = 0 nbTuFail = 0 nbTuUndef = 0 ret['testunits'] = { 'nb-pass': nbTuPass, 'nb-fail': nbTuFail, 'nb-undef': nbTuUndef } nbTaPass = 0 nbTaFail = 0 nbTaUndef = 0 ret['testabstracts'] = { 'nb-pass': nbTaPass, 'nb-fail': nbTaFail, 'nb-undef': nbTaUndef } nbTcPass = 0 nbTcFail = 0 nbTcUndef = 0 ret['testcases'] = { 'nb-pass': nbTcPass, 'nb-fail': nbTcFail, 'nb-undef': nbTcUndef } return ret self.trace('contructing tests stats') nbScriptCompleted = 0 nbScriptError = 0 nbScriptKilled = 0 try: sql_query = "SELECT SUM( IF(result = '%s', 1, 0) ) AS 'nbComplete', SUM( IF(result = '%s', 1, 0) ) AS 'nbError'," % ( TaskManager.STATE_COMPLETE, TaskManager.STATE_ERROR) sql_query += "SUM( IF(result = '%s', 1, 0) ) AS 'nbKilled' " % TaskManager.STATE_KILLED sql_query += " FROM `%s`;" % self.dbt_scripts mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbScriptCompleted = int(rows[0][0]) if rows[0][1] is not None: nbScriptError = int(rows[0][1]) if rows[0][2] is not None: nbScriptKilled = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get scripts stats: %s" % str(e)) ret['scripts'] = { 'nb-completed': nbScriptCompleted, 'nb-error': nbScriptError, 'nb-killed': nbScriptKilled } self.trace('nb-completed %s, nb-error %s, nb-killed %s' % (nbScriptCompleted, nbScriptError, nbScriptKilled)) nbTgPass = 0 nbTgFail = 0 nbTgUndef = 0 try: sql_query = "SELECT SUM( IF(result = 'PASS', 1, 0) ) AS 'nbPass', SUM( IF(result = 'FAIL', 1, 0) ) AS 'nbFail'," sql_query += "SUM( IF(result = 'UNDEFINED', 1, 0) ) AS 'nbUndef' " sql_query += " FROM `%s`;" % self.dbt_testglobals mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbTgPass = int(rows[0][0]) if rows[0][1] is not None: nbTgFail = int(rows[0][1]) if rows[0][2] is not None: nbTgUndef = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get testglobals stats: %s" % str(e)) ret['testglobals'] = { 'nb-pass': nbTgPass, 'nb-fail': nbTgFail, 'nb-undef': nbTgUndef } self.trace('nb-tg-pass %s, nb-tg-fail %s, nb-tg-undef %s' % (nbTgPass, nbTgFail, nbTgUndef)) nbTpPass = 0 nbTpFail = 0 nbTpUndef = 0 try: sql_query = "SELECT SUM( IF(result = 'PASS', 1, 0) ) AS 'nbPass', SUM( IF(result = 'FAIL', 1, 0) ) AS 'nbFail'," sql_query += "SUM( IF(result = 'UNDEFINED', 1, 0) ) AS 'nbUndef' " sql_query += " FROM `%s`;" % self.dbt_testplans mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbTpPass = int(rows[0][0]) if rows[0][1] is not None: nbTpFail = int(rows[0][1]) if rows[0][2] is not None: nbTpUndef = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get testplans stats: %s" % str(e)) ret['testplans'] = { 'nb-pass': nbTpPass, 'nb-fail': nbTpFail, 'nb-undef': nbTpUndef } self.trace('nb-tp-pass %s, nb-tp-fail %s, nb-tp-undef %s' % (nbTpPass, nbTpFail, nbTpUndef)) nbTsPass = 0 nbTsFail = 0 nbTsUndef = 0 try: sql_query = "SELECT SUM( IF(result = 'PASS', 1, 0) ) AS 'nbPass', SUM( IF(result = 'FAIL', 1, 0) ) AS 'nbFail'," sql_query += "SUM( IF(result = 'UNDEFINED', 1, 0) ) AS 'nbUndef' " sql_query += " FROM `%s`;" % self.dbt_testsuites mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbTsPass = int(rows[0][0]) if rows[0][1] is not None: nbTsFail = int(rows[0][1]) if rows[0][2] is not None: nbTsUndef = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get testsuites stats: %s" % str(e)) ret['testsuites'] = { 'nb-pass': nbTsPass, 'nb-fail': nbTsFail, 'nb-undef': nbTsUndef } self.trace('nb-ts-pass %s, nb-ts-fail %s, nb-ts-undef %s' % (nbTsPass, nbTsFail, nbTsUndef)) nbTuPass = 0 nbTuFail = 0 nbTuUndef = 0 try: sql_query = "SELECT SUM( IF(result = 'PASS', 1, 0) ) AS 'nbPass', SUM( IF(result = 'FAIL', 1, 0) ) AS 'nbFail'," sql_query += "SUM( IF(result = 'UNDEFINED', 1, 0) ) AS 'nbUndef'" sql_query += " FROM `%s`;" % self.dbt_testunits mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbTuPass = int(rows[0][0]) if rows[0][1] is not None: nbTuFail = int(rows[0][1]) if rows[0][2] is not None: nbTuUndef = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get testunits stats: %s" % str(e)) ret['testunits'] = { 'nb-pass': nbTuPass, 'nb-fail': nbTuFail, 'nb-undef': nbTuUndef } self.trace('nb-tu-pass %s, nb-tu-fail %s, nb-tu-undef %s' % (nbTuPass, nbTuFail, nbTuUndef)) nbTaPass = 0 nbTaFail = 0 nbTaUndef = 0 try: sql_query = "SELECT SUM( IF(result = 'PASS', 1, 0) ) AS 'nbPass', SUM( IF(result = 'FAIL', 1, 0) ) AS 'nbFail'," sql_query += "SUM( IF(result = 'UNDEFINED', 1, 0) ) AS 'nbUndef'" sql_query += " FROM `%s`;" % self.dbt_testabstracts mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbTaPass = int(rows[0][0]) if rows[0][1] is not None: nbTaFail = int(rows[0][1]) if rows[0][2] is not None: nbTaUndef = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get testabstracts stats: %s" % str(e)) ret['testabstracts'] = { 'nb-pass': nbTaPass, 'nb-fail': nbTaFail, 'nb-undef': nbTaUndef } self.trace('nb-ta-pass %s, nb-ta-fail %s, nb-ta-undef %s' % (nbTaPass, nbTaFail, nbTaUndef)) nbTcPass = 0 nbTcFail = 0 nbTcUndef = 0 try: sql_query = "SELECT SUM( IF(result = 'PASS', 1, 0) ) AS 'nbPass', SUM( IF(result = 'FAIL', 1, 0) ) AS 'nbFail'," sql_query += "SUM( IF(result = 'UNDEFINED', 1, 0) ) AS 'nbUndef'" sql_query += " FROM `%s`;" % self.dbt_testcases mysqlret, rows = DbManager.instance().querySQL(query=sql_query) if rows[0][0] is not None: nbTcPass = int(rows[0][0]) if rows[0][1] is not None: nbTcFail = int(rows[0][1]) if rows[0][2] is not None: nbTcUndef = int(rows[0][2]) if not mysqlret: raise Exception("sql problem") except Exception as e: self.error("unable to get testcases stats: %s" % str(e)) ret['testcases'] = { 'nb-pass': nbTcPass, 'nb-fail': nbTcFail, 'nb-undef': nbTcUndef } self.trace('nb-tc-pass %s, nb-tc-fail %s, nb-tc-undef %s' % (nbTcPass, nbTcFail, nbTcUndef)) return ret
def __init__(self): """ Documentation is generated with the start of the server in save in a file """ if Settings.getInt('Boot', 'cache-documentations'): self.generateHelps()
def getInt(section, key): """ Return value as integer according to the key """ return Settings.getInt(section, key)
def __init__(self, controllerIp, controllerPort, toolName, toolDesc, defaultTool, supportProxy=False, proxyIp=None, proxyPort=None, sslSupport=True, toolType=NetLayerLib.TYPE_AGENT_AGENT, fromCmd=False, name=None): """ Constructor for tool @param version: @type version: string """ self.__LAST_ERROR = None self.defaultTool = defaultTool self.login = "******" self.password = "******" self.toolType = toolType self.fromCmd = fromCmd # init ssl self.sslSupportFinal = sslSupport if not Settings.getBool('Server', 'ssl-support'): self.sslSupportFinal = False # init websocket wsSupport = False if Settings.getBool('Server', 'websocket-support'): wsSupport = True NetLayerLib.ClientAgent.__init__( self, typeAgent=toolType, startAuto=True, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), timeoutTcpConnect=Settings.getInt('Network', 'tcp-connect-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), sslSupport=self.sslSupportFinal, wsSupport=wsSupport, pickleVer=Settings.getInt('Network', 'pickle-version'), tcpKeepAlive=Settings.getBool('Network', 'tcp-keepalive'), tcpKeepIdle=Settings.getInt('Network', 'tcp-keepidle'), tcpKeepCnt=Settings.getInt('Network', 'tcp-keepcnt'), tcpKeepIntvl=Settings.getInt('Network', 'tcp-keepintvl')) serverPort = Settings.getInt('Server', 'port') if controllerPort is not None: serverPort = int(controllerPort) self.setServerAddress(ip=controllerIp, port=serverPort) if supportProxy: self.setProxyAddress(ip=proxyIp, port=int(proxyPort)) self.setAgentName(name=toolName) toolMore = {'details': toolDesc, 'default': defaultTool} if name is not None: self.setDetails(name=name, desc=toolMore, ver=Settings.getVersion()) else: self.setDetails(name=self.__class__.__name__, desc=toolMore, ver=Settings.getVersion()) self.controllerIp = controllerIp self.controllerPort = serverPort self.pathTraces = Settings.get('Paths', 'tmp') self.toolName = toolName self.__mutex__ = threading.RLock() self.regThread = None self.__REGISTERED_BEFORE__ = False self.regRequest = None if sys.platform == "win32": self.__tmpPath__ = "%s\\%s\\[%s]_%s\\" % (Settings.getDirExec(), self.pathTraces, toolType, self.toolName) else: self.__tmpPath__ = "%s/%s/[%s]_%s/" % (Settings.getDirExec(), self.pathTraces, toolType, self.toolName) self.__callId__ = 0 self.__callIds__ = {} self.__args__ = [] # new in v6.2 self.testsContext = {}