def onRegistration(self, client, tid, request): """ Called on the registration of a new agents @param client: @type client: @param tid: @type tid: @param request: @type request: """ self.trace("on registration") self.__mutex.acquire() doNotify = False if sys.version_info > (3, ): request['userid'] = request['userid'].decode("utf8") if request['userid'] in self.agentsRegistered: self.info('duplicate agents registration: %s' % request['userid']) NetLayerLib.ServerAgent.failed(self, client, tid) else: if not ('type' in request['body']): self.error('type missing in request: %s' % request['body']) NetLayerLib.ServerAgent.failed(self, client, tid) else: if request['body']['type'] != ClientAgent.TYPE_AGENT_AGENT: self.error('agent type refused: %s' % request['body']['type']) NetLayerLib.ServerAgent.forbidden(self, client, tid) else: tpl = { 'address': client, 'version': request['body']['version'], 'description': request['body']['description']['details'], 'auto-startup': request['body']['description']['default'], 'type': request['body']['name'], 'start-at': request['body']['start-at'], 'publicip': self.agentsPublicIp[client] } self.agentsRegistered[request['userid']] = tpl NetLayerLib.ServerAgent.ok(self, client, tid) self.info('Remote agent registered: Name="%s"' % request['userid']) doNotify = True if doNotify: # Notify all connected users notif = ('agents', ('add', self.getAgents())) ESI.instance().notifyByUserTypes(body=notif, admin=True, monitor=False, tester=True) self.__mutex.release()
def refreshTestEnvironment(self): """ Refresh test environment And notify all connected user """ for user_login, user_profile in self.usersConnected.items(): data = ('context-server', ("update", self.getInformations(user=user_login))) ESI.instance().notify(body=data, toUser=user_login) return True
def createResultLog(self, testsPath, logPath, logName, logData): """ Create result log """ self.trace("create result log=%s to %s" % (logName, logPath)) try: # write the file f = open("%s/%s/%s" % (testsPath, logPath, logName), 'wb') f.write(base64.b64decode(logData)) f.close() # notify all users size_ = os.path.getsize("%s/%s/%s" % (testsPath, logPath, logName)) # \1\2019-07-26\2019-07-26_18-31-21.1e05661b-c1a8-429c-aeee-d7646b # 87814f.MDNfMDAxIENyZWF0ZSBnb29nbGUgYWNjb3VudCBhbmQgY2FuY2Vs.admin _, projectId, mainPathTozip, subPathTozip = logPath.split(os.sep) if Settings.getInt('Notifications', 'archives'): m = [{"type": "folder", "name": mainPathTozip, "project": "%s" % projectId, "content": [{"type": "folder", "name": subPathTozip, "project": "%s" % projectId, "content": [{"project": "%s" % projectId, "type": "file", "name": logName, 'size': str(size_)}] } ] }] notif = {} notif['archive'] = m notif['stats-repo-archives'] = {'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1, 'mb-used': self.getSizeRepoV2(folder=self.testsPath), 'mb-free': self.freeSpace(p=self.testsPath)} data = ('archive', (None, notif)) ESI.instance().notifyByUserAndProject(body=data, admin=True, monitor=False, tester=True, projectId="%s" % projectId) except Exception as e: self.error("unable to create result log: %s" % e) return False return True
def unregisterChannelUser(self, login): """ Force channel disconnection """ self.info("Unregister user Login=%s" % login) UsersManager.instance().setOnlineStatus(login=login, online=False) if login not in self.usersConnected: self.trace("User=%s to unregister not found" % login) return self.CODE_NOT_FOUND user_profile = self.usersConnected[login] # close the network link with the client if exists if user_profile['address'] in ESI.instance().clients: ESI.instance().stopClient(client=user_profile['address']) else: self.usersConnected.pop(login) return self.CODE_OK
def run(self): """On run""" rsp = ESI.instance().interact(body=self.bodyReq, timeout=self.timeout) _data_ = {'cmd': Messages.CMD_INTERACT} if rsp is None: _data_['rsp'] = None else: _data_['rsp'] = rsp['body'] instance().ok(self.client, self.tid, body=_data_)
def onDisconnection(self, client): """ Reimplemented from ServerAgent @type client: @param client: """ self.trace("on disconnection") NetLayerLib.ServerAgent.onDisconnection(self, client) clientRegistered = self.agentsRegistered.items() for k, c in clientRegistered: if c['address'] == client.client_address: self.agentsRegistered.pop(k) self.agentsPublicIp.pop(c['address']) self.info('Agent unregistered: Name="%s"' % k) notif = ('agents', ('del', self.getAgents())) ESI.instance().notifyByUserTypes(body=notif, admin=True, monitor=False, tester=True) break
def zipDataV2(self, dirToday, dirTest, destPathZip, replayId, projectId=0, virtualName=""): """ Zip data by adapters and notify users in just one zip """ self.trace("Starting to zip all adapters logs") ret = False try: mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday) testDir = "%s/%s" % (mainDir, dirTest) testDir = os.path.normpath(testDir) # prepare the file name tp = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time())) \ + ".%3.3d" % int((time.time() * 1000) % 1000) fileName = "%s_%s_%s" % (self.prefixAdaptersAll, tp, replayId) # zip the folder zipped = self.zipFolder(folderPath=testDir, zipName="%s.zip" % fileName, zipPath=destPathZip) if zipped == self.context.CODE_OK: # notify users if Settings.getInt('Notifications', 'archives'): size_ = os.path.getsize("%s/%s.zip" % (destPathZip, fileName)) notif = {} m = [{ "type": "folder", "name": dirToday, "project": "%s" % projectId, "content": [{ "type": "folder", "name": dirTest, "project": "%s" % projectId, "virtual-name": virtualName, "content": [{ "type": "file", "name": "%s.zip" % fileName, 'size': str(size_), "project": "%s" % projectId }] }] }] notif['archive'] = m data = ('archive', (None, notif)) ESI.instance().notifyByUserTypes(body=data, admin=True, monitor=False, tester=True) ret = True else: self.error('error to zip data adapters') ret = False except Exception as e: self.error('unable to zip data adapters v2: %s' % str(e)) return ret
def onRequest(self, client, tid, request): """Reimplemented from ServerAgent""" self.__mutex__.acquire() try: _body_ = request['body'] if client not in self.testsConnected: self.__mutex__.release() return self.testsConnected[client]['task-id'] = _body_['task-id'] # handle notify and save some statistics on the database if request['cmd'] == Messages.RSQ_NOTIFY: try: if _body_['event'] in [ 'agent-data', 'agent-notify', 'agent-init', 'agent-reset', 'agent-alive', 'agent-ready' ]: if _body_['event'] == 'agent-ready': self.testsConnected[client]['agents'].append({ 'agent-name': _body_['destination-agent'], 'script-id': _body_['script_id'], 'uuid': _body_['uuid'], 'source-adapter': _body_['source-adapter'] }) ASI.instance().notifyAgent(client, tid, data=_body_) except Exception as e: self.error('unable to handle notify for agent: %s' % e) if _body_['event'] == 'testcase-stopped': # reset agents self.resetRunningAgent(client=self.testsConnected[client]) if _body_['task-id'] in self.tests: if not self.tests[_body_['task-id']]: # check connected time of the associated user and test # if connected-at of the user > connected-at of the test # then not necessary to send events userFounded = self.context.getUser( login=_body_['from']) if userFounded is not None: if client not in self.testsConnected: self.error('unknown test from %s' % str(client)) else: if userFounded[ 'connected-at'] < self.testsConnected[ client]['connected-at']: if _body_['channel-id']: ESI.instance().notify( body=('event', _body_), toAddress=_body_['channel-id']) else: ESI.instance().notify(body=('event', _body_)) else: self.error('test unknown: %s' % _body_['task-id']) if _body_['event'] == 'script-stopped': # reset agents self.resetRunningAgent(client=self.testsConnected[client]) if _body_['task-id'] in self.tests: self.tests.pop(_body_['task-id']) else: self.error('task-id unknown: %s' % _body_['task-id']) if client in self.testsConnected: self.testsConnected.pop(client) else: self.error('test unknown: %s' % str(client)) # handle requests elif request['cmd'] == Messages.RSQ_CMD: self.trace("cmd received: %s" % _body_['cmd']) if 'cmd' in _body_: # handle interact command if _body_['cmd'] == Messages.CMD_INTERACT: self.trace('interact called') if _body_['task-id'] in self.tests: if not self.tests[_body_['task-id']]: # check connected time of the associated user and test # if connected-at of the user > connected-at of # the test then not necessary to send events userFounded = self.context.getUser( login=_body_['from']) if userFounded is not None: if client not in self.testsConnected: self.error('unknown test from %s' % str(client)) else: if userFounded[ 'connected-at'] < self.testsConnected[ client]['connected-at']: self.__fifoThread.putItem( lambda: self.onInteract( client, tid, bodyReq=_body_, timeout=_body_['timeout'])) else: self.error('test unknown: %s' % _body_['task-id']) else: self.error('cmd unknown %s' % _body_['cmd']) rsp = {'cmd': _body_['cmd'], 'res': Messages.CMD_ERROR} NetLayerLib.ServerAgent.failed(self, client, tid, body=rsp) else: self.error('cmd is missing') # handle other request else: self.trace('%s received ' % request['cmd']) except Exception as e: self.error("unable to handle incoming request: %s" % e) self.__mutex__.release()
def cleanup(self): """ Cleanup the server """ self.info('Cleanup...') self.trace("Cleanup agent manager") try: AgentsManager.finalize() except Exception: pass self.trace("Cleanup context") try: Context.finalize() except Exception: pass self.trace("Cleanup projects manager") try: ProjectsManager.finalize() except Exception: pass self.trace("Cleanup users manager") try: UsersManager.finalize() except Exception: pass self.trace("Cleanup task manager") try: TaskManager.finalize() except Exception: pass self.trace("Cleanup test public manager") try: RepoPublic.finalize() except Exception: pass self.trace("Cleanup test repo manager") try: RepoTests.finalize() except Exception: pass self.trace("Cleanup test archives manager") try: RepoArchives.finalize() except Exception: pass self.trace("Cleanup helper manager") try: HelperManager.finalize() except Exception: pass self.trace("Cleanup adapters manager") try: RepoAdapters.finalize() except Exception: pass self.trace("Cleanup adapters data storage") try: StorageDataAdapters.finalize() except Exception: pass self.trace("Cleanup WSU") try: RestServerInterface.instance().stop() RestServerInterface.finalize() except Exception: pass self.trace("Cleanup ESI") try: EventServerInterface.instance().stopSA() EventServerInterface.finalize() except Exception: pass self.trace("Cleanup TSI") try: TestServerInterface.instance().stopSA() TestServerInterface.finalize() except Exception: pass self.trace("Cleanup ASI") try: AgentServerInterface.instance().stopSA() AgentServerInterface.finalize() except Exception: pass self.trace("Cleanup db manager") try: DbManager.finalize() except Exception: pass self.trace("Cleanup settings") try: Settings.finalize() except Exception: pass self.trace("Cleanup logger, cli") try: CliFunctions.finalize() Logger.finalize() except Exception: pass
def initialize(self): """ Starts all modules Exit if the service is alreayd running or if the config file is missing """ starttime = time.time() if self.isrunning(): sys.stdout.write(" (server is already running)") sys.exit(1) # run the server as daemon only for linux if platform.system() == "Linux": self.daemonize() try: # Initialize self.info("Starting up server...") self.trace("** System encoding (in): %s" % sys.stdin.encoding) self.trace("** System encoding (out): %s" % sys.stdout.encoding) self.info("Settings, Logger and CLI ready") DbManager.initialize() DbManager.instance().isUp() self.info("Database manager ready") # Initialize the core Context.initialize() self.info("Context ready") ProjectsManager.initialize(context=Context.instance()) self.info("Projects Manager ready") UsersManager.initialize(context=Context.instance()) self.info("Users Manager ready") TaskManager.initialize(context=Context) self.info("Task Manager ready") # Initialize all repositories RepoTests.initialize(context=Context.instance()) self.info("Repo manager for tests ready") RepoArchives.initialize(context=Context.instance()) self.info("Repo manager for archives ready") RepoAdapters.initialize(context=Context.instance()) StorageDataAdapters.initialize(context=Context.instance()) self.info("Adapters Manager and Storage Data ready") RepoPublic.initialize() self.info("Repo manager for public area is ready") HelperManager.initialize() self.info("Helper manager ready") AgentsManager.initialize(context=Context.instance()) self.info("Agents Manager ready") # Initialize all interfaces self.info("Starting ESI on %s:%s" % (Settings.get('Bind', 'ip-esi'), Settings.getInt('Bind', 'port-esi'))) EventServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-esi'), Settings.getInt( 'Bind', 'port-esi') ), sslSupport=Settings.getInt( 'Client_Channel', 'channel-ssl'), wsSupport=Settings.getInt( 'Client_Channel', 'channel-websocket-support'), context=Context.instance() ) self.info("Starting TSI on %s:%s" % (Settings.get('Bind', 'ip-tsi'), Settings.getInt('Bind', 'port-tsi'))) TestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-tsi'), Settings.getInt( 'Bind', 'port-tsi') ), context=Context.instance() ) self.info("Starting RSU on %s:%s" % (Settings.get('Bind', 'ip-rsi'), Settings.getInt('Bind', 'port-rsi'))) RestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-rsi'), Settings.getInt( 'Bind', 'port-rsi') ) ) self.info("Starting ASI on %s:%s" % (Settings.get('Bind', 'ip-asi'), Settings.getInt('Bind', 'port-asi'))) AgentServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-asi'), Settings.getInt( 'Bind', 'port-asi') ), sslSupport=Settings.getInt( 'Agent_Channel', 'channel-ssl'), wsSupport=Settings.getInt( 'Agent_Channel', 'channel-websocket-support'), tsi=TestServerInterface.instance(), context=Context.instance() ) # Start on modules RestServerInterface.instance().start() self.info("RSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-rsi'), Settings.get('Bind', 'port-rsi'))) EventServerInterface.instance().startSA() self.info("ESI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-esi'), Settings.get('Bind', 'port-esi'))) TestServerInterface.instance().startSA() self.info("TSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-tsi'), Settings.get('Bind', 'port-tsi'))) AgentServerInterface.instance().startSA() self.info("ASI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-asi'), Settings.get('Bind', 'port-asi'))) # Now start the scheduler and reload tasks taskReloaded = TaskManager.instance().loadBackups() if taskReloaded is None: self.info("Reload tasks disabled") elif taskReloaded: self.info("Tasks reloaded") else: self.error("Failed to reload tasks") except Exception as e: self.error("Unable to start server: %s" % str(e)) self.cleanup() sys.exit(3) stoptime = time.time() self.info("%s successfully started (in %s sec.)" % (Settings.get('Server', 'name'), int(stoptime - starttime))) # only for windows platform if platform.system() == "Windows": print("Server successfully started...") self.setrunning() self.run()
def updateLinkedScriptPath(self, project, mainPath, oldFilename, extFilename, newProject, newPath, newFilename, newExt, user_login, file_referer_path='', file_referer_projectid=0): """ Fix linked test in testplan or testglobal Pull request by dbr13 and updated by dmachard """ # get current project name and accessible projects list for currnet # user project_name = ProjectsManager.instance().getProjectName(prjId=project) new_project_name = ProjectsManager.instance().getProjectName( prjId=newProject) projects = ProjectsManager.instance().getProjects(user=user_login) updated_files_list = [] new_test_file_name = '%s:/%s/%s.%s' % (new_project_name, newPath, newFilename, newExt) old_test_file_name = '%s:/%s/%s.%s' % (project_name, mainPath, oldFilename, extFilename) self.trace("Update link - test path to search: %s" % old_test_file_name) self.trace("Update link - replace the old one by the new path: %s" % new_test_file_name) for proj_id in projects: project_id = proj_id['project_id'] _, _, listing, _ = self.getTree(project=project_id) tests_tree_update_locations = self.getTestsForUpdate( listing=listing, extFileName=extFilename) if len(file_referer_path) and int( proj_id['project_id']) == int(file_referer_projectid): files_paths = self.get_files_paths( tests_tree=tests_tree_update_locations, exceptions=[file_referer_path]) else: files_paths = self.get_files_paths( tests_tree=tests_tree_update_locations) for file_path in files_paths: # init appropriate data model for current file path if file_path.endswith(RepoManager.TEST_PLAN_EXT): doc = TestPlan.DataModel() ext_file_name = RepoManager.TEST_PLAN_EXT elif file_path.endswith(RepoManager.TEST_GLOBAL_EXT): doc = TestPlan.DataModel(isGlobal=True) ext_file_name = RepoManager.TEST_GLOBAL_EXT else: return "Bad file extension: %s" % file_path absPath = '%s%s%s' % (self.testsPath, project_id, file_path) doc.load(absPath=absPath) test_files_list = doc.testplan['testplan']['testfile'] is_changed = False for test_file in test_files_list: if os.path.normpath( old_test_file_name) == os.path.normpath( test_file['file']): test_file['file'] = new_test_file_name test_file['extension'] = extFilename is_changed = True if is_changed: file_content = doc.getRaw() f_path_list = file_path.split('/') path_file = '/'.join(f_path_list[:-1]) name_file = f_path_list[-1][:-4] path_file = os.path.normpath(path_file) self.uploadFile(pathFile=path_file, nameFile=name_file, extFile=ext_file_name, contentFile=file_content, login=user_login, project=project_id, overwriteFile=True, createFolders=False, lockMode=False, binaryMode=True, closeAfter=False) # notify all connected users of the change data = ('test', ("changed", { "modified-by": user_login, "path": file_path, "project-id": project_id })) ESI.instance().notifyByUserAndProject( body=data, admin=True, monitor=True, tester=True, projectId=int(project_id)) # append the file modified to the list updated_files_list.append(file_path) return updated_files_list