def trace(self, txt): """ Trace message """ if Settings.instance() is not None: if Settings.get('Trace', 'debug-level') == 'VERBOSE': Logger.ClassLogger.trace(self, txt=txt)
def stopping(self): """ On stopping the server """ self.info("Stopping server...") # cleanup all child processes for f in os.listdir("%s/%s" % (Settings.getDirExec(), Settings.get('Paths', 'run'))): if f.endswith(".pid"): pid = f.split(".pid")[0] # kill the process if pid.isdigit(): self.info('Stopping chid processes %s...' % pid) try: while True: os.kill(int(pid), signal.SIGTERM) time.sleep(0.1) except OSError: pass time.sleep(1) # just to be sure, delete a second time try: os.remove("%s/%s/%s.pid" % (Settings.getDirExec(), Settings.get('Paths', 'run'), pid)) except Exception: pass
def addPyInitFile(self, pathFile, descr="", helper="", allmodules="", adps=False, mainInit=False): """ Add the default __init__ file of the repository """ HEADER = '' tpl_path = "%s/%s/adapter_header.tpl" % ( Settings.getDirExec(), Settings.get('Paths', 'templates')) try: fd = open(tpl_path, "r") HEADER = fd.read() fd.close() except Exception as e: self.error('unable to read template adapter header: %s' % str(e)) try: if mainInit: default_init = MAIN_INIT % (HEADER, descr, helper, allmodules) else: default_init = ADP_INIT % (HEADER, descr) f = open('%s/__init__.py' % pathFile, 'w') f.write(default_init) f.close() except Exception as e: self.error(e) return False return True
def __init__(self): """ Repository manager for public test files """ RepoManager.RepoManager.__init__( self, pathRepo='%s%s' % (Settings.getDirExec(), Settings.get('Paths', 'public')), extensionsSupported=[])
def isUp(self): """ Try to connect to the database Detect the version of the mysql server """ db_name = "%s/%s/%s" % (Settings.getDirExec(), Settings.get('Paths', 'var'), Settings.get('Database', 'db')) sqlite3.connect(db_name) self.trace("database connection successful")
def __init__(self, listeningAddress, agentName='ESI', sslSupport=False, wsSupport=False, context=None): """ Event server interface @param listeningAddress: @type listeningAddress: @param agentName: agent name used on request @type agentName: string """ NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), sslSupport=sslSupport, wsSupport=wsSupport, certFile='%s/%s' % (Settings.getDirExec(), Settings.get('Client_Channel', 'channel-ssl-cert')), keyFile='%s/%s' % (Settings.getDirExec(), Settings.get('Client_Channel', 'channel-ssl-key')), pickleVer=Settings.getInt('Network', 'pickle-version')) self.__mutex__ = threading.RLock() self.context = context
def getStaticArgs(envTmp=False): """ """ te_args = """root = r'%s/../' tests_result_path = r'%s' controller_ip = '%s' controller_port = %s """ % (os.path.normpath( Settings.getDirExec()), os.path.normpath(getTestsPath(envTmp=envTmp)), Settings.get('Bind', 'ip-tsi'), Settings.get('Bind', 'port-tsi')) return te_args
def __init__(self): """ """ threading.Thread.__init__(self) self.event = threading.Event() self.mutex = threading.RLock() self.running = True self.lease = int(Settings.get('Users_Session', 'max-expiry-age')) # in seconds self.expire = int(Settings.get('Users_Session', 'timeout-cleanup')) # in seconds
def doLdapAuth(self, login, password): """ perform bind ldap authentication with multiple ldaps server and ssl mode """ auth_success = False # get ldap settings ldap_host_list = json.loads(Settings.get('Users_Session', 'ldap-host')) ldap_dn_list = json.loads(Settings.get('Users_Session', 'ldap-dn')) # define ldap server(s) servers_list = [] for host in ldap_host_list: use_ssl = False ldap_port = 386 # parse the url to extract scheme host and port url_parsed = urllib.parse.urlparse(host) if url_parsed.scheme == "ldaps": use_ssl = True ldap_port = 636 if ":" in url_parsed.netloc: ldap_host, ldap_port = url_parsed.netloc.split(":") else: ldap_host = url_parsed.netloc server = ldap3.Server(ldap_host, port=int(ldap_port), use_ssl=use_ssl) servers_list.append(server) last_auth_err = "" for bind_dn in ldap_dn_list: c = ldap3.Connection(servers_list, user=bind_dn % login, password=password) # perform the Bind operation auth_success = c.bind() last_auth_err = c.result if auth_success: break if not auth_success: self.trace(last_auth_err) return auth_success
def __init__(self, context): """ Storage data adapters """ RepoManager.RepoManager.__init__( self, pathRepo='%s%s' % (Settings.getDirExec(), Settings.get('Paths', 'tmp')), context=context) self.context = context self.prefixAdapters = "adapter" self.prefixAdaptersAll = "private_storage" self.adpDataPath = os.path.normpath("%s/AdaptersData" % self.testsPath) self.initStorage()
def __init__(self, context): """ Repository manager for archives files """ RepoManager.RepoManager.__init__(self, pathRepo='%s%s' % (Settings.getDirExec(), Settings.get('Paths', 'testsresults')), extensionsSupported=[RepoManager.TEST_RESULT_EXT, RepoManager.TXT_EXT, RepoManager.CAP_EXT, RepoManager.ZIP_EXT, RepoManager.PNG_EXT, RepoManager.JPG_EXT], context=context) self.context = context self.cacheUuids = {} self.cachingUuid() self.trace("nb entries in testresult cache: %s" % len(self.cacheUuids))
def __init__(self, context): """ Construct Adpaters Manager """ RepoManager.RepoManager.__init__( self, pathRepo='%s/%s/' % (Settings.getDirExec(), Settings.get('Paths', 'packages-sutadapters')), extensionsSupported=[RepoManager.PY_EXT, RepoManager.TXT_EXT], context=context) self.context = context # update main init file self.updateMainInit()
def getTestsPath(envTmp=False): """ Get the path of all tests result @return: @rtype: string """ if envTmp: trPath = '%s%s' % (Settings.getDirExec(), Settings.get('Paths', 'testsresults-tmp')) else: trPath = '%s%s' % (Settings.getDirExec(), Settings.get('Paths', 'testsresults')) # normalize the path and return it return os.path.normpath(trPath)
def __init__(self, context): """ Class Projects Manager """ self.tb_projects = 'projects' self.repoTests = '%s/%s' % (Settings.getDirExec(), Settings.get('Paths', 'tests')) self.context = context # load projects in cache, new in v19 self.__cache = [] self.loadCache() # Initialize the repository self.info('Deploying folders projects and reserved folders...') self.createDirProjects()
def __init__(self, listeningAddress, agentName='TSI', context=None): """Constructs TCP Server Inferface""" NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), pickleVer=Settings.getInt('Network', 'pickle-version')) self.context = context self.__mutex__ = threading.RLock() self.__fifoThread = None self.tests = {} # {'task-id': Boolean} # test register, # with background running or not self.testsConnected = {} # all tests connected
def getHelps(self): """ Returns the documentation cache """ self.trace("get helps") ret = '' try: complete_path = '%s/%s/documentations.dat' % (Settings.getDirExec(), Settings.get('Paths', 'var')) if os.path.exists(complete_path): fd = open(complete_path, "rb") data = fd.read() fd.close() ret = base64.b64encode(zlib.compress(data)) else: self.error('documentation cache does not exist') except Exception as e: self.error("unable to get helps: %s" % e) return ret.decode('utf8')
def hupHandler(self, signum, frame): """ Hup handler """ self.info('Reloading configuration...') # reload settings ini Settings.finalize() Settings.initialize() # reload config from database Context.instance().readConfigDb() # reconfigure the level of log message Logger.reconfigureLevel() # reload cache UsersManager.instance().loadCache() self.info('Configuration reloaded!')
def __init__(self, context): """ Repository manager for tests files """ RepoManager.RepoManager.__init__( self, pathRepo='%s%s' % (Settings.getDirExec(), Settings.get('Paths', 'tests')), extensionsSupported=[ RepoManager.TEST_SUITE_EXT, RepoManager.TEST_PLAN_EXT, RepoManager.TEST_CONFIG_EXT, RepoManager.TEST_DATA_EXT, RepoManager.TEST_UNIT_EXT, RepoManager.PNG_EXT, #RepoManager.TEST_YAML_EXT, RepoManager.TEST_GLOBAL_EXT ], context=context) self.context = context
def updateSession(self, sessionId): """ """ if sessionId in self.userSessions: self.userSessions[sessionId]['last_activity'] = time.time() lease = int(Settings.get('Users_Session', 'max-expiry-age')) # in seconds end = time.gmtime(self.userSessions[sessionId]['last_activity'] + lease) expires = time.strftime("%a, %d-%b-%Y %T GMT", end) return expires return ''
def installAdapter(self, name): """ """ RepoAdapters.initialize(context=None) folder_lib = "%s/%s/%s" % (Settings.getDirExec(), Settings.get('Paths', 'packages-sutadapters'), name) folder_lib = os.path.normpath(folder_lib) if os.path.exists(folder_lib): try: # install dependancies according to the plugin with pip pip_list = "%s/deps/pip_list.txt" % folder_lib pip_list = os.path.normpath(pip_list) if os.path.exists(pip_list) and os.path.getsize(pip_list) > 0: if platform.system() == "Windows": cmd = "%s -m pip install -r \"%s\"" % (Settings.get( 'Bin', 'python-win'), pip_list) else: cmd = "%s -m pip install -r %s" % (Settings.get( 'Bin', 'python'), pip_list) subprocess.call(cmd, shell=True) # system detect yum_list = "%s/deps/yum_list.txt" % folder_lib yum_list = os.path.normpath(yum_list) if os.path.exists(yum_list) and os.path.getsize(yum_list) > 0: if os.path.exists("/etc/os-release"): os_id = "" with open("/etc/os-release") as f: for line in f: if "=" in line: k, v = line.rstrip().split("=") if k == "ID": os_id = v.strip('"') break if "centos" in os_id or "rhel" in os_id: cmd = "yum install `cat %s | tr '\n' ' '`" % yum_list subprocess.call(cmd, shell=True) RepoAdapters.instance().updateMainInit() # install samples according to the plugin if os.path.exists("%s/samples/" % folder_lib): folder_sample = "%s/%s/1/Samples/Adapter_%s" % ( Settings.getDirExec(), Settings.get('Paths', 'tests'), name) shutil.copytree("%s/samples/" % folder_lib, folder_sample) print("Sut Adapter installation process terminated") except Exception as e: print("unable to install adapter: %s" % e) else: print("Sut Adapter (%s) not found!" % name) RepoAdapters.instance().updateMainInit()
def __init__(self, listeningAddress, agentName='ASI', sslSupport=False, wsSupport=False, tsi=None, context=None): """ Construct Agent Server Interface @param listeningAddress: @type listeningAddress: @param agentName: @type agentName: string """ NetLayerLib.ServerAgent.__init__( self, listeningAddress=listeningAddress, agentName=agentName, keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'), inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'), responseTimeout=Settings.getInt('Network', 'response-timeout'), selectTimeout=Settings.get('Network', 'select-timeout'), sslSupport=sslSupport, wsSupport=wsSupport, certFile='%s/%s' % (Settings.getDirExec(), Settings.get('Agent_Channel', 'channel-ssl-cert')), keyFile='%s/%s' % (Settings.getDirExec(), Settings.get('Agent_Channel', 'channel-ssl-key')), pickleVer=Settings.getInt('Network', 'pickle-version')) self.tsi = tsi self.context = context self.__mutex = threading.RLock() self.__mutexNotif = threading.RLock() self.agentsRegistered = {} self.agentsPublicIp = {}
def createResultLog(self, testsPath, logPath, logName, logData): """ Create result log """ self.trace("create result log=%s to %s" % (logName, logPath)) try: # write the file f = open("%s/%s/%s" % (testsPath, logPath, logName), 'wb') f.write(base64.b64decode(logData)) f.close() # notify all users size_ = os.path.getsize("%s/%s/%s" % (testsPath, logPath, logName)) # \1\2019-07-26\2019-07-26_18-31-21.1e05661b-c1a8-429c-aeee-d7646b # 87814f.MDNfMDAxIENyZWF0ZSBnb29nbGUgYWNjb3VudCBhbmQgY2FuY2Vs.admin _, projectId, mainPathTozip, subPathTozip = logPath.split(os.sep) if Settings.getInt('Notifications', 'archives'): m = [{"type": "folder", "name": mainPathTozip, "project": "%s" % projectId, "content": [{"type": "folder", "name": subPathTozip, "project": "%s" % projectId, "content": [{"project": "%s" % projectId, "type": "file", "name": logName, 'size': str(size_)}] } ] }] notif = {} notif['archive'] = m notif['stats-repo-archives'] = {'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1, 'mb-used': self.getSizeRepoV2(folder=self.testsPath), 'mb-free': self.freeSpace(p=self.testsPath)} data = ('archive', (None, notif)) ESI.instance().notifyByUserAndProject(body=data, admin=True, monitor=False, tester=True, projectId="%s" % projectId) except Exception as e: self.error("unable to create result log: %s" % e) return False return True
def reconfigureLevel(): """ Reconfigure the level log """ try: global LG level = Settings.get(section='Trace', key='level') if level == 'DEBUG': # write everything messages LG.setLevel(logging.DEBUG) elif level == 'ERROR': # write anything that is an error or worse. LG.setLevel(logging.ERROR) elif level == 'INFO': # write anything that is an info message or worse. LG.setLevel(logging.INFO) except Exception as e: sys.stdout.write("error: %s" % e)
def getTree(self, b64=False, fullTree=False, project=1): """ Return tree of files """ nb = Settings.getInt('WebServices', 'nb-archives') if nb == -1: nb = None if nb == 0: return (0, 0, [], {}) if fullTree: nb = None success = os.path.exists("%s/%s" % (self.testsPath, project)) if not success: return (0, 0, [], {}) else: return self.getListingFilesV2(path="%s/%s" % (self.testsPath, project), nbDirs=nb, project=project, archiveMode=True)
def initialize(logPathFile=None, level="INFO", size="5", nbFiles="10", noSettings=False): """ Initialize """ global LG if not noSettings: if logPathFile is not None: file = logPathFile else: file = "%s/%s/%s" % (Settings.getDirExec(), Settings.get(section='Paths', key='logs'), Settings.get(section='Trace', key='file')) level = Settings.get(section='Trace', key='level') size = Settings.get(section='Trace', key='max-size-file') maxBytes = int(size.split('M')[0]) * 1024 * 1024 nbFilesMax = Settings.getInt(section='Trace', key='nb-backup-max') else: file = logPathFile level = level size = size maxBytes = size nbFilesMax = nbFiles LG = logging.getLogger('Logger') if level == 'DEBUG': # write everything messages LG.setLevel(logging.DEBUG) elif level == 'ERROR': # write anything that is an error or worse. LG.setLevel(logging.ERROR) elif level == 'INFO': # write anything that is an info message or worse. LG.setLevel(logging.INFO) handler = logging.handlers.RotatingFileHandler(file, maxBytes=maxBytes, backupCount=nbFilesMax) # format='%(asctime)-6s: %(name)s - %(levelname)s - %(module)s - # %(funcName)s - %(lineno)d - %(message)s', formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) LG.addHandler(handler)
def getInformations(self, user): """ Returns settings on the server for the client """ self.trace('construct servers information') ret = [] # get project and default one according to the user if isinstance(user, UserContext): user_projects = user.getProjects() user_def_project = user.getDefault() else: user_projects = ProjectsManager.instance().getProjects(user=user) user_def_project = ProjectsManager.instance( ).getDefaultProjectForUser(user=user) # append informations in the list to return ret.append({'version': Settings.getVersion()}) ret.append({'python': platform.python_version()}) ret.append({'test-environment': self.getTestEnvironment(user=user)}) ret.append({'projects': user_projects}) ret.append({'default-project': user_def_project}) return ret
def apiAuthorization(self, login, password): """ Check authorization for rest api """ self.trace('Rest authorization called for Login=%s' % (login)) expires = '' # check if this login exists on the database cache_users = UsersManager.instance().cache() if login not in cache_users: self.trace("Login=%s account not found" % login) return (self.CODE_NOT_FOUND, expires) user_profile = cache_users[login] # account disable ? if not user_profile['active']: self.trace("%s account not active" % login) return (self.CODE_DISABLED, expires) # 2 methods to authenticate the user # make a hash of the password and look inside the server # or communicate with a remote ldap authenticator if Settings.getInt('Users_Session', 'ldap-authbind') and LDAP_INSTALLED: auth_success = self.doLdapAuth(login, password) if not auth_success: self.trace("ldap auth failed for %s account" % login) return (self.CODE_FAILED, expires) elif Settings.getInt('Users_Session', 'ldap-authbind') and not LDAP_INSTALLED: self.error("python ldap3 library is not installed on your system") return (self.CODE_FAILED, expires) else: # check password, create a sha1 hash with salt: sha1(salt + # sha1(password)) sha0 = hashlib.sha1() sha0.update(password.encode('utf8')) sha1 = hashlib.sha1() _pwd = "%s%s" % (self.cfg_db["auth-salt"], sha0.hexdigest()) sha1.update(_pwd.encode('utf8')) sha3 = hashlib.sha1() _pwd2 = "%s%s" % (self.cfg_db["auth-salt"], password.encode('utf8')) sha3.update(_pwd2.encode('utf8')) pwd_matched = False if user_profile['password'] == sha1.hexdigest(): pwd_matched = True # keep this mode only for backward compatibility if user_profile['password'] == sha3.hexdigest(): pwd_matched = True if not pwd_matched: self.trace("incorrect password for %s account" % login) return (self.CODE_FAILED, expires) session_id = self.generateSessionid() user_profile['last_activity'] = time.time() lease = int(Settings.get('Users_Session', 'max-expiry-age')) # in seconds end = time.gmtime(user_profile['last_activity'] + lease) expires = time.strftime("%a, %d-%b-%Y %T GMT", end) self.userSessions.update({session_id: user_profile}) self.trace('Rest authorized for Login=%s SessionId=%s Expires=%s' % (login, session_id, expires)) return (session_id, expires)
def version(self): """ Get version of the server """ sys.stdout.write("Server version: %s\n" % Settings.getVersion())
import platform import shutil try: import cStringIO except ImportError: # support python 3 import io as cStringIO try: import cPickle except ImportError: # support python 3 import pickle as cPickle from ea.libs import Settings, Logger from ea.libs.FileModels import TestResult as TestResult from ea.serverrepositories import (RepoAdapters) Settings.initialize() def querySQL(query, db): """ """ try: conn = sqlite3.connect(db) c = conn.cursor() c.execute(query) c.close() conn.commit() conn.close() except Exception as e:
def finalize(): """ Finalize settings """ Settings.finalize()