def __init__(self,
listeningAddress,
agentName='ESI',
sslSupport=False,
wsSupport=False,
context=None):
"""
Event server interface
@param listeningAddress:
@type listeningAddress:
@param agentName: agent name used on request
@type agentName: string
"""
NetLayerLib.ServerAgent.__init__(
self,
listeningAddress=listeningAddress,
agentName=agentName,
keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'),
inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'),
responseTimeout=Settings.getInt('Network', 'response-timeout'),
selectTimeout=Settings.get('Network', 'select-timeout'),
sslSupport=sslSupport,
wsSupport=wsSupport,
certFile='%s/%s' %
(Settings.getDirExec(),
Settings.get('Client_Channel', 'channel-ssl-cert')),
keyFile='%s/%s' %
(Settings.getDirExec(),
Settings.get('Client_Channel', 'channel-ssl-key')),
pickleVer=Settings.getInt('Network', 'pickle-version'))
self.__mutex__ = threading.RLock()
self.context = context
def __init__(self, listeningAddress, agentName='TSI', context=None):
"""Constructs TCP Server Inferface"""
NetLayerLib.ServerAgent.__init__(
self,
listeningAddress=listeningAddress,
agentName=agentName,
keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'),
inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'),
responseTimeout=Settings.getInt('Network', 'response-timeout'),
selectTimeout=Settings.get('Network', 'select-timeout'),
pickleVer=Settings.getInt('Network', 'pickle-version'))
self.context = context
self.__mutex__ = threading.RLock()
self.__fifoThread = None
self.tests = {} # {'task-id': Boolean} # test register,
# with background running or not
self.testsConnected = {} # all tests connected
def __init__(self,
listeningAddress,
agentName='ASI',
sslSupport=False,
wsSupport=False,
tsi=None,
context=None):
"""
Construct Agent Server Interface
@param listeningAddress:
@type listeningAddress:
@param agentName:
@type agentName: string
"""
NetLayerLib.ServerAgent.__init__(
self,
listeningAddress=listeningAddress,
agentName=agentName,
keepAliveInterval=Settings.getInt('Network', 'keepalive-interval'),
inactivityTimeout=Settings.getInt('Network', 'inactivity-timeout'),
responseTimeout=Settings.getInt('Network', 'response-timeout'),
selectTimeout=Settings.get('Network', 'select-timeout'),
sslSupport=sslSupport,
wsSupport=wsSupport,
certFile='%s/%s' %
(Settings.getDirExec(),
Settings.get('Agent_Channel', 'channel-ssl-cert')),
keyFile='%s/%s' %
(Settings.getDirExec(),
Settings.get('Agent_Channel', 'channel-ssl-key')),
pickleVer=Settings.getInt('Network', 'pickle-version'))
self.tsi = tsi
self.context = context
self.__mutex = threading.RLock()
self.__mutexNotif = threading.RLock()
self.agentsRegistered = {}
self.agentsPublicIp = {}
def initialize(logPathFile=None,
level="INFO",
size="5",
nbFiles="10",
noSettings=False):
"""
Initialize
"""
global LG
if not noSettings:
if logPathFile is not None:
file = logPathFile
else:
file = "%s/%s/%s" % (Settings.getDirExec(),
Settings.get(section='Paths', key='logs'),
Settings.get(section='Trace', key='file'))
level = Settings.get(section='Trace', key='level')
size = Settings.get(section='Trace', key='max-size-file')
maxBytes = int(size.split('M')[0]) * 1024 * 1024
nbFilesMax = Settings.getInt(section='Trace', key='nb-backup-max')
else:
file = logPathFile
level = level
size = size
maxBytes = size
nbFilesMax = nbFiles
LG = logging.getLogger('Logger')
if level == 'DEBUG':
# write everything messages
LG.setLevel(logging.DEBUG)
elif level == 'ERROR':
# write anything that is an error or worse.
LG.setLevel(logging.ERROR)
elif level == 'INFO':
# write anything that is an info message or worse.
LG.setLevel(logging.INFO)
handler = logging.handlers.RotatingFileHandler(file,
maxBytes=maxBytes,
backupCount=nbFilesMax)
# format='%(asctime)-6s: %(name)s - %(levelname)s - %(module)s -
# %(funcName)s - %(lineno)d - %(message)s',
formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
LG.addHandler(handler)
def createResultLog(self, testsPath, logPath, logName, logData):
"""
Create result log
"""
self.trace("create result log=%s to %s" % (logName, logPath))
try:
# write the file
f = open("%s/%s/%s" % (testsPath, logPath, logName), 'wb')
f.write(base64.b64decode(logData))
f.close()
# notify all users
size_ = os.path.getsize("%s/%s/%s" % (testsPath, logPath, logName))
# \1\2019-07-26\2019-07-26_18-31-21.1e05661b-c1a8-429c-aeee-d7646b
# 87814f.MDNfMDAxIENyZWF0ZSBnb29nbGUgYWNjb3VudCBhbmQgY2FuY2Vs.admin
_, projectId, mainPathTozip, subPathTozip = logPath.split(os.sep)
if Settings.getInt('Notifications', 'archives'):
m = [{"type": "folder", "name": mainPathTozip, "project": "%s" % projectId,
"content": [{"type": "folder",
"name": subPathTozip,
"project": "%s" % projectId,
"content": [{"project": "%s" % projectId,
"type": "file",
"name": logName,
'size': str(size_)}]
}
]
}]
notif = {}
notif['archive'] = m
notif['stats-repo-archives'] = {'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1,
'mb-used': self.getSizeRepoV2(folder=self.testsPath),
'mb-free': self.freeSpace(p=self.testsPath)}
data = ('archive', (None, notif))
ESI.instance().notifyByUserAndProject(body=data,
admin=True,
monitor=False,
tester=True,
projectId="%s" % projectId)
except Exception as e:
self.error("unable to create result log: %s" % e)
return False
return True
def getTree(self, b64=False, fullTree=False, project=1):
"""
Return tree of files
"""
nb = Settings.getInt('WebServices', 'nb-archives')
if nb == -1:
nb = None
if nb == 0:
return (0, 0, [], {})
if fullTree:
nb = None
success = os.path.exists("%s/%s" % (self.testsPath, project))
if not success:
return (0, 0, [], {})
else:
return self.getListingFilesV2(path="%s/%s" % (self.testsPath, project),
nbDirs=nb,
project=project,
archiveMode=True)
def apiAuthorization(self, login, password):
"""
Check authorization for rest api
"""
self.trace('Rest authorization called for Login=%s' % (login))
expires = ''
# check if this login exists on the database
cache_users = UsersManager.instance().cache()
if login not in cache_users:
self.trace("Login=%s account not found" % login)
return (self.CODE_NOT_FOUND, expires)
user_profile = cache_users[login]
# account disable ?
if not user_profile['active']:
self.trace("%s account not active" % login)
return (self.CODE_DISABLED, expires)
# 2 methods to authenticate the user
# make a hash of the password and look inside the server
# or communicate with a remote ldap authenticator
if Settings.getInt('Users_Session',
'ldap-authbind') and LDAP_INSTALLED:
auth_success = self.doLdapAuth(login, password)
if not auth_success:
self.trace("ldap auth failed for %s account" % login)
return (self.CODE_FAILED, expires)
elif Settings.getInt('Users_Session',
'ldap-authbind') and not LDAP_INSTALLED:
self.error("python ldap3 library is not installed on your system")
return (self.CODE_FAILED, expires)
else:
# check password, create a sha1 hash with salt: sha1(salt +
# sha1(password))
sha0 = hashlib.sha1()
sha0.update(password.encode('utf8'))
sha1 = hashlib.sha1()
_pwd = "%s%s" % (self.cfg_db["auth-salt"], sha0.hexdigest())
sha1.update(_pwd.encode('utf8'))
sha3 = hashlib.sha1()
_pwd2 = "%s%s" % (self.cfg_db["auth-salt"],
password.encode('utf8'))
sha3.update(_pwd2.encode('utf8'))
pwd_matched = False
if user_profile['password'] == sha1.hexdigest():
pwd_matched = True
# keep this mode only for backward compatibility
if user_profile['password'] == sha3.hexdigest():
pwd_matched = True
if not pwd_matched:
self.trace("incorrect password for %s account" % login)
return (self.CODE_FAILED, expires)
session_id = self.generateSessionid()
user_profile['last_activity'] = time.time()
lease = int(Settings.get('Users_Session',
'max-expiry-age')) # in seconds
end = time.gmtime(user_profile['last_activity'] + lease)
expires = time.strftime("%a, %d-%b-%Y %T GMT", end)
self.userSessions.update({session_id: user_profile})
self.trace('Rest authorized for Login=%s SessionId=%s Expires=%s' %
(login, session_id, expires))
return (session_id, expires)
def getInt(section, key):
"""
Return value as integer according to the key
"""
return Settings.getInt(section, key)
def zipDataV2(self,
dirToday,
dirTest,
destPathZip,
replayId,
projectId=0,
virtualName=""):
"""
Zip data by adapters and notify users in just one zip
"""
self.trace("Starting to zip all adapters logs")
ret = False
try:
mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday)
testDir = "%s/%s" % (mainDir, dirTest)
testDir = os.path.normpath(testDir)
# prepare the file name
tp = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time())) \
+ ".%3.3d" % int((time.time() * 1000) % 1000)
fileName = "%s_%s_%s" % (self.prefixAdaptersAll, tp, replayId)
# zip the folder
zipped = self.zipFolder(folderPath=testDir,
zipName="%s.zip" % fileName,
zipPath=destPathZip)
if zipped == self.context.CODE_OK:
# notify users
if Settings.getInt('Notifications', 'archives'):
size_ = os.path.getsize("%s/%s.zip" %
(destPathZip, fileName))
notif = {}
m = [{
"type":
"folder",
"name":
dirToday,
"project":
"%s" % projectId,
"content": [{
"type":
"folder",
"name":
dirTest,
"project":
"%s" % projectId,
"virtual-name":
virtualName,
"content": [{
"type": "file",
"name": "%s.zip" % fileName,
'size': str(size_),
"project": "%s" % projectId
}]
}]
}]
notif['archive'] = m
data = ('archive', (None, notif))
ESI.instance().notifyByUserTypes(body=data,
admin=True,
monitor=False,
tester=True)
ret = True
else:
self.error('error to zip data adapters')
ret = False
except Exception as e:
self.error('unable to zip data adapters v2: %s' % str(e))
return ret
def initialize(self):
"""
Starts all modules
Exit if the service is alreayd running or if the config file is missing
"""
starttime = time.time()
if self.isrunning():
sys.stdout.write(" (server is already running)")
sys.exit(1)
# run the server as daemon only for linux
if platform.system() == "Linux":
self.daemonize()
try:
# Initialize
self.info("Starting up server...")
self.trace("** System encoding (in): %s" % sys.stdin.encoding)
self.trace("** System encoding (out): %s" % sys.stdout.encoding)
self.info("Settings, Logger and CLI ready")
DbManager.initialize()
DbManager.instance().isUp()
self.info("Database manager ready")
# Initialize the core
Context.initialize()
self.info("Context ready")
ProjectsManager.initialize(context=Context.instance())
self.info("Projects Manager ready")
UsersManager.initialize(context=Context.instance())
self.info("Users Manager ready")
TaskManager.initialize(context=Context)
self.info("Task Manager ready")
# Initialize all repositories
RepoTests.initialize(context=Context.instance())
self.info("Repo manager for tests ready")
RepoArchives.initialize(context=Context.instance())
self.info("Repo manager for archives ready")
RepoAdapters.initialize(context=Context.instance())
StorageDataAdapters.initialize(context=Context.instance())
self.info("Adapters Manager and Storage Data ready")
RepoPublic.initialize()
self.info("Repo manager for public area is ready")
HelperManager.initialize()
self.info("Helper manager ready")
AgentsManager.initialize(context=Context.instance())
self.info("Agents Manager ready")
# Initialize all interfaces
self.info("Starting ESI on %s:%s" % (Settings.get('Bind', 'ip-esi'),
Settings.getInt('Bind', 'port-esi')))
EventServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-esi'),
Settings.getInt(
'Bind', 'port-esi')
),
sslSupport=Settings.getInt(
'Client_Channel', 'channel-ssl'),
wsSupport=Settings.getInt(
'Client_Channel', 'channel-websocket-support'),
context=Context.instance()
)
self.info("Starting TSI on %s:%s" % (Settings.get('Bind', 'ip-tsi'),
Settings.getInt('Bind', 'port-tsi')))
TestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-tsi'),
Settings.getInt(
'Bind', 'port-tsi')
),
context=Context.instance()
)
self.info("Starting RSU on %s:%s" % (Settings.get('Bind', 'ip-rsi'),
Settings.getInt('Bind', 'port-rsi')))
RestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-rsi'),
Settings.getInt(
'Bind', 'port-rsi')
)
)
self.info("Starting ASI on %s:%s" % (Settings.get('Bind', 'ip-asi'),
Settings.getInt('Bind', 'port-asi')))
AgentServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-asi'),
Settings.getInt(
'Bind', 'port-asi')
),
sslSupport=Settings.getInt(
'Agent_Channel', 'channel-ssl'),
wsSupport=Settings.getInt(
'Agent_Channel', 'channel-websocket-support'),
tsi=TestServerInterface.instance(),
context=Context.instance()
)
# Start on modules
RestServerInterface.instance().start()
self.info("RSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-rsi'),
Settings.get('Bind', 'port-rsi')))
EventServerInterface.instance().startSA()
self.info("ESI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-esi'),
Settings.get('Bind', 'port-esi')))
TestServerInterface.instance().startSA()
self.info("TSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-tsi'),
Settings.get('Bind', 'port-tsi')))
AgentServerInterface.instance().startSA()
self.info("ASI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-asi'),
Settings.get('Bind', 'port-asi')))
# Now start the scheduler and reload tasks
taskReloaded = TaskManager.instance().loadBackups()
if taskReloaded is None:
self.info("Reload tasks disabled")
elif taskReloaded:
self.info("Tasks reloaded")
else:
self.error("Failed to reload tasks")
except Exception as e:
self.error("Unable to start server: %s" % str(e))
self.cleanup()
sys.exit(3)
stoptime = time.time()
self.info("%s successfully started (in %s sec.)" % (Settings.get('Server', 'name'),
int(stoptime - starttime)))
# only for windows platform
if platform.system() == "Windows":
print("Server successfully started...")
self.setrunning()
self.run()