def __init__(self, listeningAddress, agentName='ESI', sslSupport=False, wsSupport=False, context=None): """ Event server interface @param listeningAddress: @type listeningAddress: @param agentName: agent name used on request @type agentName: string """ NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), sslSupport=sslSupport, wsSupport=wsSupport, certFile='%s/%s' % (Settings.getDirExec(), Settings.get('Client_Channel', 'channel-ssl-cert')), keyFile='%s/%s' % (Settings.getDirExec(), Settings.get('Client_Channel', 'channel-ssl-key')), pickleVer=Settings.getInt('Network', 'pickle-version')) self.__mutex__ = threading.RLock() self.context = context
def __init__(self, listeningAddress, agentName='TSI', context=None): """Constructs TCP Server Inferface""" NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), pickleVer=Settings.getInt('Network', 'pickle-version')) self.context = context self.__mutex__ = threading.RLock() self.__fifoThread = None self.tests = {} # {'task-id': Boolean} # test register, # with background running or not self.testsConnected = {} # all tests connected
def __init__(self, listeningAddress, agentName='ASI', sslSupport=False, wsSupport=False, tsi=None, context=None): """ Construct Agent Server Interface @param listeningAddress: @type listeningAddress: @param agentName: @type agentName: string """ NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), sslSupport=sslSupport, wsSupport=wsSupport, certFile='%s/%s' % (Settings.getDirExec(), Settings.get('Agent_Channel', 'channel-ssl-cert')), keyFile='%s/%s' % (Settings.getDirExec(), Settings.get('Agent_Channel', 'channel-ssl-key')), pickleVer=Settings.getInt('Network', 'pickle-version')) self.tsi = tsi self.context = context self.__mutex = threading.RLock() self.__mutexNotif = threading.RLock() self.agentsRegistered = {} self.agentsPublicIp = {}
def initialize(logPathFile=None, level="INFO", size="5", nbFiles="10", noSettings=False): """ Initialize """ global LG if not noSettings: if logPathFile is not None: file = logPathFile else: file = "%s/%s/%s" % (Settings.getDirExec(), Settings.get(section='Paths', key='logs'), Settings.get(section='Trace', key='file')) level = Settings.get(section='Trace', key='level') size = Settings.get(section='Trace', key='max-size-file') maxBytes = int(size.split('M')[0]) * 1024 * 1024 nbFilesMax = Settings.getInt(section='Trace', key='nb-backup-max') else: file = logPathFile level = level size = size maxBytes = size nbFilesMax = nbFiles LG = logging.getLogger('Logger') if level == 'DEBUG': # write everything messages LG.setLevel(logging.DEBUG) elif level == 'ERROR': # write anything that is an error or worse. LG.setLevel(logging.ERROR) elif level == 'INFO': # write anything that is an info message or worse. LG.setLevel(logging.INFO) handler = logging.handlers.RotatingFileHandler(file, maxBytes=maxBytes, backupCount=nbFilesMax) # format='%(asctime)-6s: %(name)s - %(levelname)s - %(module)s - # %(funcName)s - %(lineno)d - %(message)s', formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) LG.addHandler(handler)
def createResultLog(self, testsPath, logPath, logName, logData): """ Create result log """ self.trace("create result log=%s to %s" % (logName, logPath)) try: # write the file f = open("%s/%s/%s" % (testsPath, logPath, logName), 'wb') f.write(base64.b64decode(logData)) f.close() # notify all users size_ = os.path.getsize("%s/%s/%s" % (testsPath, logPath, logName)) # \1\2019-07-26\2019-07-26_18-31-21.1e05661b-c1a8-429c-aeee-d7646b # 87814f.MDNfMDAxIENyZWF0ZSBnb29nbGUgYWNjb3VudCBhbmQgY2FuY2Vs.admin _, projectId, mainPathTozip, subPathTozip = logPath.split(os.sep) if Settings.getInt('Notifications', 'archives'): m = [{"type": "folder", "name": mainPathTozip, "project": "%s" % projectId, "content": [{"type": "folder", "name": subPathTozip, "project": "%s" % projectId, "content": [{"project": "%s" % projectId, "type": "file", "name": logName, 'size': str(size_)}] } ] }] notif = {} notif['archive'] = m notif['stats-repo-archives'] = {'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1, 'mb-used': self.getSizeRepoV2(folder=self.testsPath), 'mb-free': self.freeSpace(p=self.testsPath)} data = ('archive', (None, notif)) ESI.instance().notifyByUserAndProject(body=data, admin=True, monitor=False, tester=True, projectId="%s" % projectId) except Exception as e: self.error("unable to create result log: %s" % e) return False return True
def getTree(self, b64=False, fullTree=False, project=1): """ Return tree of files """ nb = Settings.getInt('WebServices', 'nb-archives') if nb == -1: nb = None if nb == 0: return (0, 0, [], {}) if fullTree: nb = None success = os.path.exists("%s/%s" % (self.testsPath, project)) if not success: return (0, 0, [], {}) else: return self.getListingFilesV2(path="%s/%s" % (self.testsPath, project), nbDirs=nb, project=project, archiveMode=True)
def apiAuthorization(self, login, password): """ Check authorization for rest api """ self.trace('Rest authorization called for Login=%s' % (login)) expires = '' # check if this login exists on the database cache_users = UsersManager.instance().cache() if login not in cache_users: self.trace("Login=%s account not found" % login) return (self.CODE_NOT_FOUND, expires) user_profile = cache_users[login] # account disable ? if not user_profile['active']: self.trace("%s account not active" % login) return (self.CODE_DISABLED, expires) # 2 methods to authenticate the user # make a hash of the password and look inside the server # or communicate with a remote ldap authenticator if Settings.getInt('Users_Session', 'ldap-authbind') and LDAP_INSTALLED: auth_success = self.doLdapAuth(login, password) if not auth_success: self.trace("ldap auth failed for %s account" % login) return (self.CODE_FAILED, expires) elif Settings.getInt('Users_Session', 'ldap-authbind') and not LDAP_INSTALLED: self.error("python ldap3 library is not installed on your system") return (self.CODE_FAILED, expires) else: # check password, create a sha1 hash with salt: sha1(salt + # sha1(password)) sha0 = hashlib.sha1() sha0.update(password.encode('utf8')) sha1 = hashlib.sha1() _pwd = "%s%s" % (self.cfg_db["auth-salt"], sha0.hexdigest()) sha1.update(_pwd.encode('utf8')) sha3 = hashlib.sha1() _pwd2 = "%s%s" % (self.cfg_db["auth-salt"], password.encode('utf8')) sha3.update(_pwd2.encode('utf8')) pwd_matched = False if user_profile['password'] == sha1.hexdigest(): pwd_matched = True # keep this mode only for backward compatibility if user_profile['password'] == sha3.hexdigest(): pwd_matched = True if not pwd_matched: self.trace("incorrect password for %s account" % login) return (self.CODE_FAILED, expires) session_id = self.generateSessionid() user_profile['last_activity'] = time.time() lease = int(Settings.get('Users_Session', 'max-expiry-age')) # in seconds end = time.gmtime(user_profile['last_activity'] + lease) expires = time.strftime("%a, %d-%b-%Y %T GMT", end) self.userSessions.update({session_id: user_profile}) self.trace('Rest authorized for Login=%s SessionId=%s Expires=%s' % (login, session_id, expires)) return (session_id, expires)
def getInt(section, key): """ Return value as integer according to the key """ return Settings.getInt(section, key)
def zipDataV2(self, dirToday, dirTest, destPathZip, replayId, projectId=0, virtualName=""): """ Zip data by adapters and notify users in just one zip """ self.trace("Starting to zip all adapters logs") ret = False try: mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday) testDir = "%s/%s" % (mainDir, dirTest) testDir = os.path.normpath(testDir) # prepare the file name tp = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time())) \ + ".%3.3d" % int((time.time() * 1000) % 1000) fileName = "%s_%s_%s" % (self.prefixAdaptersAll, tp, replayId) # zip the folder zipped = self.zipFolder(folderPath=testDir, zipName="%s.zip" % fileName, zipPath=destPathZip) if zipped == self.context.CODE_OK: # notify users if Settings.getInt('Notifications', 'archives'): size_ = os.path.getsize("%s/%s.zip" % (destPathZip, fileName)) notif = {} m = [{ "type": "folder", "name": dirToday, "project": "%s" % projectId, "content": [{ "type": "folder", "name": dirTest, "project": "%s" % projectId, "virtual-name": virtualName, "content": [{ "type": "file", "name": "%s.zip" % fileName, 'size': str(size_), "project": "%s" % projectId }] }] }] notif['archive'] = m data = ('archive', (None, notif)) ESI.instance().notifyByUserTypes(body=data, admin=True, monitor=False, tester=True) ret = True else: self.error('error to zip data adapters') ret = False except Exception as e: self.error('unable to zip data adapters v2: %s' % str(e)) return ret
def initialize(self): """ Starts all modules Exit if the service is alreayd running or if the config file is missing """ starttime = time.time() if self.isrunning(): sys.stdout.write(" (server is already running)") sys.exit(1) # run the server as daemon only for linux if platform.system() == "Linux": self.daemonize() try: # Initialize self.info("Starting up server...") self.trace("** System encoding (in): %s" % sys.stdin.encoding) self.trace("** System encoding (out): %s" % sys.stdout.encoding) self.info("Settings, Logger and CLI ready") DbManager.initialize() DbManager.instance().isUp() self.info("Database manager ready") # Initialize the core Context.initialize() self.info("Context ready") ProjectsManager.initialize(context=Context.instance()) self.info("Projects Manager ready") UsersManager.initialize(context=Context.instance()) self.info("Users Manager ready") TaskManager.initialize(context=Context) self.info("Task Manager ready") # Initialize all repositories RepoTests.initialize(context=Context.instance()) self.info("Repo manager for tests ready") RepoArchives.initialize(context=Context.instance()) self.info("Repo manager for archives ready") RepoAdapters.initialize(context=Context.instance()) StorageDataAdapters.initialize(context=Context.instance()) self.info("Adapters Manager and Storage Data ready") RepoPublic.initialize() self.info("Repo manager for public area is ready") HelperManager.initialize() self.info("Helper manager ready") AgentsManager.initialize(context=Context.instance()) self.info("Agents Manager ready") # Initialize all interfaces self.info("Starting ESI on %s:%s" % (Settings.get('Bind', 'ip-esi'), Settings.getInt('Bind', 'port-esi'))) EventServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-esi'), Settings.getInt( 'Bind', 'port-esi') ), sslSupport=Settings.getInt( 'Client_Channel', 'channel-ssl'), wsSupport=Settings.getInt( 'Client_Channel', 'channel-websocket-support'), context=Context.instance() ) self.info("Starting TSI on %s:%s" % (Settings.get('Bind', 'ip-tsi'), Settings.getInt('Bind', 'port-tsi'))) TestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-tsi'), Settings.getInt( 'Bind', 'port-tsi') ), context=Context.instance() ) self.info("Starting RSU on %s:%s" % (Settings.get('Bind', 'ip-rsi'), Settings.getInt('Bind', 'port-rsi'))) RestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-rsi'), Settings.getInt( 'Bind', 'port-rsi') ) ) self.info("Starting ASI on %s:%s" % (Settings.get('Bind', 'ip-asi'), Settings.getInt('Bind', 'port-asi'))) AgentServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-asi'), Settings.getInt( 'Bind', 'port-asi') ), sslSupport=Settings.getInt( 'Agent_Channel', 'channel-ssl'), wsSupport=Settings.getInt( 'Agent_Channel', 'channel-websocket-support'), tsi=TestServerInterface.instance(), context=Context.instance() ) # Start on modules RestServerInterface.instance().start() self.info("RSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-rsi'), Settings.get('Bind', 'port-rsi'))) EventServerInterface.instance().startSA() self.info("ESI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-esi'), Settings.get('Bind', 'port-esi'))) TestServerInterface.instance().startSA() self.info("TSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-tsi'), Settings.get('Bind', 'port-tsi'))) AgentServerInterface.instance().startSA() self.info("ASI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-asi'), Settings.get('Bind', 'port-asi'))) # Now start the scheduler and reload tasks taskReloaded = TaskManager.instance().loadBackups() if taskReloaded is None: self.info("Reload tasks disabled") elif taskReloaded: self.info("Tasks reloaded") else: self.error("Failed to reload tasks") except Exception as e: self.error("Unable to start server: %s" % str(e)) self.cleanup() sys.exit(3) stoptime = time.time() self.info("%s successfully started (in %s sec.)" % (Settings.get('Server', 'name'), int(stoptime - starttime))) # only for windows platform if platform.system() == "Windows": print("Server successfully started...") self.setrunning() self.run()