def onRegistration(self, client, tid, request):
"""
Called on the registration of a new agents
@param client:
@type client:
@param tid:
@type tid:
@param request:
@type request:
"""
self.trace("on registration")
self.__mutex.acquire()
doNotify = False
if sys.version_info > (3, ):
request['userid'] = request['userid'].decode("utf8")
if request['userid'] in self.agentsRegistered:
self.info('duplicate agents registration: %s' % request['userid'])
NetLayerLib.ServerAgent.failed(self, client, tid)
else:
if not ('type' in request['body']):
self.error('type missing in request: %s' % request['body'])
NetLayerLib.ServerAgent.failed(self, client, tid)
else:
if request['body']['type'] != ClientAgent.TYPE_AGENT_AGENT:
self.error('agent type refused: %s' %
request['body']['type'])
NetLayerLib.ServerAgent.forbidden(self, client, tid)
else:
tpl = {
'address': client,
'version': request['body']['version'],
'description':
request['body']['description']['details'],
'auto-startup':
request['body']['description']['default'],
'type': request['body']['name'],
'start-at': request['body']['start-at'],
'publicip': self.agentsPublicIp[client]
}
self.agentsRegistered[request['userid']] = tpl
NetLayerLib.ServerAgent.ok(self, client, tid)
self.info('Remote agent registered: Name="%s"' %
request['userid'])
doNotify = True
if doNotify:
# Notify all connected users
notif = ('agents', ('add', self.getAgents()))
ESI.instance().notifyByUserTypes(body=notif,
admin=True,
monitor=False,
tester=True)
self.__mutex.release()
def refreshTestEnvironment(self):
"""
Refresh test environment
And notify all connected user
"""
for user_login, user_profile in self.usersConnected.items():
data = ('context-server', ("update",
self.getInformations(user=user_login)))
ESI.instance().notify(body=data, toUser=user_login)
return True
def createResultLog(self, testsPath, logPath, logName, logData):
"""
Create result log
"""
self.trace("create result log=%s to %s" % (logName, logPath))
try:
# write the file
f = open("%s/%s/%s" % (testsPath, logPath, logName), 'wb')
f.write(base64.b64decode(logData))
f.close()
# notify all users
size_ = os.path.getsize("%s/%s/%s" % (testsPath, logPath, logName))
# \1\2019-07-26\2019-07-26_18-31-21.1e05661b-c1a8-429c-aeee-d7646b
# 87814f.MDNfMDAxIENyZWF0ZSBnb29nbGUgYWNjb3VudCBhbmQgY2FuY2Vs.admin
_, projectId, mainPathTozip, subPathTozip = logPath.split(os.sep)
if Settings.getInt('Notifications', 'archives'):
m = [{"type": "folder", "name": mainPathTozip, "project": "%s" % projectId,
"content": [{"type": "folder",
"name": subPathTozip,
"project": "%s" % projectId,
"content": [{"project": "%s" % projectId,
"type": "file",
"name": logName,
'size': str(size_)}]
}
]
}]
notif = {}
notif['archive'] = m
notif['stats-repo-archives'] = {'nb-zip': 1, 'nb-trx': 0, 'nb-tot': 1,
'mb-used': self.getSizeRepoV2(folder=self.testsPath),
'mb-free': self.freeSpace(p=self.testsPath)}
data = ('archive', (None, notif))
ESI.instance().notifyByUserAndProject(body=data,
admin=True,
monitor=False,
tester=True,
projectId="%s" % projectId)
except Exception as e:
self.error("unable to create result log: %s" % e)
return False
return True
def unregisterChannelUser(self, login):
"""
Force channel disconnection
"""
self.info("Unregister user Login=%s" % login)
UsersManager.instance().setOnlineStatus(login=login, online=False)
if login not in self.usersConnected:
self.trace("User=%s to unregister not found" % login)
return self.CODE_NOT_FOUND
user_profile = self.usersConnected[login]
# close the network link with the client if exists
if user_profile['address'] in ESI.instance().clients:
ESI.instance().stopClient(client=user_profile['address'])
else:
self.usersConnected.pop(login)
return self.CODE_OK
def run(self):
"""On run"""
rsp = ESI.instance().interact(body=self.bodyReq, timeout=self.timeout)
_data_ = {'cmd': Messages.CMD_INTERACT}
if rsp is None:
_data_['rsp'] = None
else:
_data_['rsp'] = rsp['body']
instance().ok(self.client, self.tid, body=_data_)
def onDisconnection(self, client):
"""
Reimplemented from ServerAgent
@type client:
@param client:
"""
self.trace("on disconnection")
NetLayerLib.ServerAgent.onDisconnection(self, client)
clientRegistered = self.agentsRegistered.items()
for k, c in clientRegistered:
if c['address'] == client.client_address:
self.agentsRegistered.pop(k)
self.agentsPublicIp.pop(c['address'])
self.info('Agent unregistered: Name="%s"' % k)
notif = ('agents', ('del', self.getAgents()))
ESI.instance().notifyByUserTypes(body=notif,
admin=True,
monitor=False,
tester=True)
break
def zipDataV2(self,
dirToday,
dirTest,
destPathZip,
replayId,
projectId=0,
virtualName=""):
"""
Zip data by adapters and notify users in just one zip
"""
self.trace("Starting to zip all adapters logs")
ret = False
try:
mainDir = "%s/%s/%s" % (self.adpDataPath, projectId, dirToday)
testDir = "%s/%s" % (mainDir, dirTest)
testDir = os.path.normpath(testDir)
# prepare the file name
tp = time.strftime("%Y-%m-%d_%H-%M-%S", time.localtime(time.time())) \
+ ".%3.3d" % int((time.time() * 1000) % 1000)
fileName = "%s_%s_%s" % (self.prefixAdaptersAll, tp, replayId)
# zip the folder
zipped = self.zipFolder(folderPath=testDir,
zipName="%s.zip" % fileName,
zipPath=destPathZip)
if zipped == self.context.CODE_OK:
# notify users
if Settings.getInt('Notifications', 'archives'):
size_ = os.path.getsize("%s/%s.zip" %
(destPathZip, fileName))
notif = {}
m = [{
"type":
"folder",
"name":
dirToday,
"project":
"%s" % projectId,
"content": [{
"type":
"folder",
"name":
dirTest,
"project":
"%s" % projectId,
"virtual-name":
virtualName,
"content": [{
"type": "file",
"name": "%s.zip" % fileName,
'size': str(size_),
"project": "%s" % projectId
}]
}]
}]
notif['archive'] = m
data = ('archive', (None, notif))
ESI.instance().notifyByUserTypes(body=data,
admin=True,
monitor=False,
tester=True)
ret = True
else:
self.error('error to zip data adapters')
ret = False
except Exception as e:
self.error('unable to zip data adapters v2: %s' % str(e))
return ret
def onRequest(self, client, tid, request):
"""Reimplemented from ServerAgent"""
self.__mutex__.acquire()
try:
_body_ = request['body']
if client not in self.testsConnected:
self.__mutex__.release()
return
self.testsConnected[client]['task-id'] = _body_['task-id']
# handle notify and save some statistics on the database
if request['cmd'] == Messages.RSQ_NOTIFY:
try:
if _body_['event'] in [
'agent-data', 'agent-notify', 'agent-init',
'agent-reset', 'agent-alive', 'agent-ready'
]:
if _body_['event'] == 'agent-ready':
self.testsConnected[client]['agents'].append({
'agent-name':
_body_['destination-agent'],
'script-id':
_body_['script_id'],
'uuid':
_body_['uuid'],
'source-adapter':
_body_['source-adapter']
})
ASI.instance().notifyAgent(client, tid, data=_body_)
except Exception as e:
self.error('unable to handle notify for agent: %s' % e)
if _body_['event'] == 'testcase-stopped':
# reset agents
self.resetRunningAgent(client=self.testsConnected[client])
if _body_['task-id'] in self.tests:
if not self.tests[_body_['task-id']]:
# check connected time of the associated user and test
# if connected-at of the user > connected-at of the test
# then not necessary to send events
userFounded = self.context.getUser(
login=_body_['from'])
if userFounded is not None:
if client not in self.testsConnected:
self.error('unknown test from %s' %
str(client))
else:
if userFounded[
'connected-at'] < self.testsConnected[
client]['connected-at']:
if _body_['channel-id']:
ESI.instance().notify(
body=('event', _body_),
toAddress=_body_['channel-id'])
else:
ESI.instance().notify(body=('event',
_body_))
else:
self.error('test unknown: %s' % _body_['task-id'])
if _body_['event'] == 'script-stopped':
# reset agents
self.resetRunningAgent(client=self.testsConnected[client])
if _body_['task-id'] in self.tests:
self.tests.pop(_body_['task-id'])
else:
self.error('task-id unknown: %s' % _body_['task-id'])
if client in self.testsConnected:
self.testsConnected.pop(client)
else:
self.error('test unknown: %s' % str(client))
# handle requests
elif request['cmd'] == Messages.RSQ_CMD:
self.trace("cmd received: %s" % _body_['cmd'])
if 'cmd' in _body_:
# handle interact command
if _body_['cmd'] == Messages.CMD_INTERACT:
self.trace('interact called')
if _body_['task-id'] in self.tests:
if not self.tests[_body_['task-id']]:
# check connected time of the associated user and test
# if connected-at of the user > connected-at of
# the test then not necessary to send events
userFounded = self.context.getUser(
login=_body_['from'])
if userFounded is not None:
if client not in self.testsConnected:
self.error('unknown test from %s' %
str(client))
else:
if userFounded[
'connected-at'] < self.testsConnected[
client]['connected-at']:
self.__fifoThread.putItem(
lambda: self.onInteract(
client,
tid,
bodyReq=_body_,
timeout=_body_['timeout']))
else:
self.error('test unknown: %s' % _body_['task-id'])
else:
self.error('cmd unknown %s' % _body_['cmd'])
rsp = {'cmd': _body_['cmd'], 'res': Messages.CMD_ERROR}
NetLayerLib.ServerAgent.failed(self,
client,
tid,
body=rsp)
else:
self.error('cmd is missing')
# handle other request
else:
self.trace('%s received ' % request['cmd'])
except Exception as e:
self.error("unable to handle incoming request: %s" % e)
self.__mutex__.release()
def cleanup(self):
"""
Cleanup the server
"""
self.info('Cleanup...')
self.trace("Cleanup agent manager")
try:
AgentsManager.finalize()
except Exception:
pass
self.trace("Cleanup context")
try:
Context.finalize()
except Exception:
pass
self.trace("Cleanup projects manager")
try:
ProjectsManager.finalize()
except Exception:
pass
self.trace("Cleanup users manager")
try:
UsersManager.finalize()
except Exception:
pass
self.trace("Cleanup task manager")
try:
TaskManager.finalize()
except Exception:
pass
self.trace("Cleanup test public manager")
try:
RepoPublic.finalize()
except Exception:
pass
self.trace("Cleanup test repo manager")
try:
RepoTests.finalize()
except Exception:
pass
self.trace("Cleanup test archives manager")
try:
RepoArchives.finalize()
except Exception:
pass
self.trace("Cleanup helper manager")
try:
HelperManager.finalize()
except Exception:
pass
self.trace("Cleanup adapters manager")
try:
RepoAdapters.finalize()
except Exception:
pass
self.trace("Cleanup adapters data storage")
try:
StorageDataAdapters.finalize()
except Exception:
pass
self.trace("Cleanup WSU")
try:
RestServerInterface.instance().stop()
RestServerInterface.finalize()
except Exception:
pass
self.trace("Cleanup ESI")
try:
EventServerInterface.instance().stopSA()
EventServerInterface.finalize()
except Exception:
pass
self.trace("Cleanup TSI")
try:
TestServerInterface.instance().stopSA()
TestServerInterface.finalize()
except Exception:
pass
self.trace("Cleanup ASI")
try:
AgentServerInterface.instance().stopSA()
AgentServerInterface.finalize()
except Exception:
pass
self.trace("Cleanup db manager")
try:
DbManager.finalize()
except Exception:
pass
self.trace("Cleanup settings")
try:
Settings.finalize()
except Exception:
pass
self.trace("Cleanup logger, cli")
try:
CliFunctions.finalize()
Logger.finalize()
except Exception:
pass
def initialize(self):
"""
Starts all modules
Exit if the service is alreayd running or if the config file is missing
"""
starttime = time.time()
if self.isrunning():
sys.stdout.write(" (server is already running)")
sys.exit(1)
# run the server as daemon only for linux
if platform.system() == "Linux":
self.daemonize()
try:
# Initialize
self.info("Starting up server...")
self.trace("** System encoding (in): %s" % sys.stdin.encoding)
self.trace("** System encoding (out): %s" % sys.stdout.encoding)
self.info("Settings, Logger and CLI ready")
DbManager.initialize()
DbManager.instance().isUp()
self.info("Database manager ready")
# Initialize the core
Context.initialize()
self.info("Context ready")
ProjectsManager.initialize(context=Context.instance())
self.info("Projects Manager ready")
UsersManager.initialize(context=Context.instance())
self.info("Users Manager ready")
TaskManager.initialize(context=Context)
self.info("Task Manager ready")
# Initialize all repositories
RepoTests.initialize(context=Context.instance())
self.info("Repo manager for tests ready")
RepoArchives.initialize(context=Context.instance())
self.info("Repo manager for archives ready")
RepoAdapters.initialize(context=Context.instance())
StorageDataAdapters.initialize(context=Context.instance())
self.info("Adapters Manager and Storage Data ready")
RepoPublic.initialize()
self.info("Repo manager for public area is ready")
HelperManager.initialize()
self.info("Helper manager ready")
AgentsManager.initialize(context=Context.instance())
self.info("Agents Manager ready")
# Initialize all interfaces
self.info("Starting ESI on %s:%s" % (Settings.get('Bind', 'ip-esi'),
Settings.getInt('Bind', 'port-esi')))
EventServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-esi'),
Settings.getInt(
'Bind', 'port-esi')
),
sslSupport=Settings.getInt(
'Client_Channel', 'channel-ssl'),
wsSupport=Settings.getInt(
'Client_Channel', 'channel-websocket-support'),
context=Context.instance()
)
self.info("Starting TSI on %s:%s" % (Settings.get('Bind', 'ip-tsi'),
Settings.getInt('Bind', 'port-tsi')))
TestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-tsi'),
Settings.getInt(
'Bind', 'port-tsi')
),
context=Context.instance()
)
self.info("Starting RSU on %s:%s" % (Settings.get('Bind', 'ip-rsi'),
Settings.getInt('Bind', 'port-rsi')))
RestServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-rsi'),
Settings.getInt(
'Bind', 'port-rsi')
)
)
self.info("Starting ASI on %s:%s" % (Settings.get('Bind', 'ip-asi'),
Settings.getInt('Bind', 'port-asi')))
AgentServerInterface.initialize(listeningAddress=(Settings.get('Bind', 'ip-asi'),
Settings.getInt(
'Bind', 'port-asi')
),
sslSupport=Settings.getInt(
'Agent_Channel', 'channel-ssl'),
wsSupport=Settings.getInt(
'Agent_Channel', 'channel-websocket-support'),
tsi=TestServerInterface.instance(),
context=Context.instance()
)
# Start on modules
RestServerInterface.instance().start()
self.info("RSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-rsi'),
Settings.get('Bind', 'port-rsi')))
EventServerInterface.instance().startSA()
self.info("ESI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-esi'),
Settings.get('Bind', 'port-esi')))
TestServerInterface.instance().startSA()
self.info("TSI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-tsi'),
Settings.get('Bind', 'port-tsi')))
AgentServerInterface.instance().startSA()
self.info("ASI is listening on tcp://%s:%s" % (Settings.get('Bind', 'ip-asi'),
Settings.get('Bind', 'port-asi')))
# Now start the scheduler and reload tasks
taskReloaded = TaskManager.instance().loadBackups()
if taskReloaded is None:
self.info("Reload tasks disabled")
elif taskReloaded:
self.info("Tasks reloaded")
else:
self.error("Failed to reload tasks")
except Exception as e:
self.error("Unable to start server: %s" % str(e))
self.cleanup()
sys.exit(3)
stoptime = time.time()
self.info("%s successfully started (in %s sec.)" % (Settings.get('Server', 'name'),
int(stoptime - starttime)))
# only for windows platform
if platform.system() == "Windows":
print("Server successfully started...")
self.setrunning()
self.run()
def updateLinkedScriptPath(self,
project,
mainPath,
oldFilename,
extFilename,
newProject,
newPath,
newFilename,
newExt,
user_login,
file_referer_path='',
file_referer_projectid=0):
"""
Fix linked test in testplan or testglobal
Pull request by dbr13 and updated by dmachard
"""
# get current project name and accessible projects list for currnet
# user
project_name = ProjectsManager.instance().getProjectName(prjId=project)
new_project_name = ProjectsManager.instance().getProjectName(
prjId=newProject)
projects = ProjectsManager.instance().getProjects(user=user_login)
updated_files_list = []
new_test_file_name = '%s:/%s/%s.%s' % (new_project_name, newPath,
newFilename, newExt)
old_test_file_name = '%s:/%s/%s.%s' % (project_name, mainPath,
oldFilename, extFilename)
self.trace("Update link - test path to search: %s" %
old_test_file_name)
self.trace("Update link - replace the old one by the new path: %s" %
new_test_file_name)
for proj_id in projects:
project_id = proj_id['project_id']
_, _, listing, _ = self.getTree(project=project_id)
tests_tree_update_locations = self.getTestsForUpdate(
listing=listing, extFileName=extFilename)
if len(file_referer_path) and int(
proj_id['project_id']) == int(file_referer_projectid):
files_paths = self.get_files_paths(
tests_tree=tests_tree_update_locations,
exceptions=[file_referer_path])
else:
files_paths = self.get_files_paths(
tests_tree=tests_tree_update_locations)
for file_path in files_paths:
# init appropriate data model for current file path
if file_path.endswith(RepoManager.TEST_PLAN_EXT):
doc = TestPlan.DataModel()
ext_file_name = RepoManager.TEST_PLAN_EXT
elif file_path.endswith(RepoManager.TEST_GLOBAL_EXT):
doc = TestPlan.DataModel(isGlobal=True)
ext_file_name = RepoManager.TEST_GLOBAL_EXT
else:
return "Bad file extension: %s" % file_path
absPath = '%s%s%s' % (self.testsPath, project_id, file_path)
doc.load(absPath=absPath)
test_files_list = doc.testplan['testplan']['testfile']
is_changed = False
for test_file in test_files_list:
if os.path.normpath(
old_test_file_name) == os.path.normpath(
test_file['file']):
test_file['file'] = new_test_file_name
test_file['extension'] = extFilename
is_changed = True
if is_changed:
file_content = doc.getRaw()
f_path_list = file_path.split('/')
path_file = '/'.join(f_path_list[:-1])
name_file = f_path_list[-1][:-4]
path_file = os.path.normpath(path_file)
self.uploadFile(pathFile=path_file,
nameFile=name_file,
extFile=ext_file_name,
contentFile=file_content,
login=user_login,
project=project_id,
overwriteFile=True,
createFolders=False,
lockMode=False,
binaryMode=True,
closeAfter=False)
# notify all connected users of the change
data = ('test', ("changed", {
"modified-by": user_login,
"path": file_path,
"project-id": project_id
}))
ESI.instance().notifyByUserAndProject(
body=data,
admin=True,
monitor=True,
tester=True,
projectId=int(project_id))
# append the file modified to the list
updated_files_list.append(file_path)
return updated_files_list