def WaitBuildeNBisFinished(self): if self.eNB_serverId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) count = 40 buildOAIprocess = True while (count > 0) and buildOAIprocess: mySSH.command('ps aux | grep --color=never build_ | grep -v grep', '\$', 6) result = re.search('build_oai', mySSH.getBefore()) if result is None: buildOAIprocess = False else: count -= 1 time.sleep(30) mySSH.close() self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.backgroundBuildTestId[int(self.eNB_instance)])
def InitializeMME(self, HTML): if self.IPAddress == '' or self.UserName == '' or self.Password == '' or self.SourceCodePath == '' or self.Type == '': HELP.GenericHelp(CONST.Version) HELP.EPCSrvHelp(self.IPAddress, self.UserName, self.Password, self.SourceCodePath, self.Type) sys.exit('Insufficient EPC Parameters') mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 MME in Docker') mySSH.command( 'docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup tshark -i eth0 -i lo:s10 -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 5) time.sleep(5) mySSH.command( 'docker exec -d ' + self.containerPrefix + '-oai-mme /bin/bash -c "nohup ./bin/oai_mme -c ./etc/' + self.mmeConfFile + ' > mme_check_run.log 2>&1"', '\$', 5) elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 MME') mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command( 'echo ' + self.Password + ' | sudo -S rm -f mme_' + self.testCase_id + '.log', '\$', 5) mySSH.command( 'echo "./run_mme --config-file /usr/local/etc/oai/mme.conf --set-virt-if" > ./my-mme.sh', '\$', 5) mySSH.command('chmod 755 ./my-mme.sh', '\$', 5) mySSH.command( 'sudo daemon --unsafe --name=mme_daemon --chdir=' + self.SourceCodePath + '/scripts -o ' + self.SourceCodePath + '/scripts/mme_' + self.testCase_id + '.log ./my-mme.sh', '\$', 5) elif re.match('OAI', self.Type, re.IGNORECASE): mySSH.command('cd ' + self.SourceCodePath, '\$', 5) mySSH.command('source oaienv', '\$', 5) mySSH.command('cd scripts', '\$', 5) mySSH.command('stdbuf -o0 hostname', '\$', 5) result = re.search( 'hostname\\\\r\\\\n(?P<host_name>[a-zA-Z0-9\-\_]+)\\\\r\\\\n', mySSH.getBefore()) if result is None: logging.debug('\u001B[1;37;41m Hostname Not Found! \u001B[0m') sys.exit(1) host_name = result.group('host_name') mySSH.command( 'echo ' + self.Password + ' | sudo -S ./run_mme 2>&1 | stdbuf -o0 tee -a mme_' + self.testCase_id + '.log &', 'MME app initialization complete', 100) elif re.match('ltebox', self.Type, re.IGNORECASE): mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./start_mme', '\$', 5) else: logging.error('This option should not occur!') mySSH.close() HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
def InitializeHSS(self): if self.IPAddress == '' or self.UserName == '' or self.Password == '' or self.SourceCodePath == '' or self.Type == '': HELP.GenericHelp(CONST.Version) HELP.EPCSrvHelp(self.IPAddress, self.UserName, self.Password, self.SourceCodePath, self.Type) sys.exit('Insufficient EPC Parameters') mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 Cassandra-based HSS in Docker') mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-hss /bin/bash -c "nohup tshark -i eth0 -i eth1 -w /tmp/hss_check_run.pcap 2>&1 > /dev/null"', '\$', 5) time.sleep(5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-hss /bin/bash -c "nohup ./bin/oai_hss -j ./etc/hss_rel14.json --reloadkey true > hss_check_run.log 2>&1"', '\$', 5) elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 Cassandra-based HSS') mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5) logging.debug('\u001B[1m Launching tshark on all interfaces \u001B[0m') self.PcapFileName = 'epc_' + self.testCase_id + '.pcap' mySSH.command('echo ' + self.Password + ' | sudo -S rm -f ' + self.PcapFileName, '\$', 5) mySSH.command('echo $USER; nohup sudo tshark -f "tcp port not 22 and port not 53" -i any -w ' + self.SourceCodePath + '/scripts/' + self.PcapFileName + ' > /tmp/tshark.log 2>&1 &', self.UserName, 5) mySSH.command('echo ' + self.Password + ' | sudo -S mkdir -p logs', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S rm -f hss_' + self.testCase_id + '.log logs/hss*.*', '\$', 5) mySSH.command('echo "oai_hss -j /usr/local/etc/oai/hss_rel14.json" > ./my-hss.sh', '\$', 5) mySSH.command('chmod 755 ./my-hss.sh', '\$', 5) mySSH.command('sudo daemon --unsafe --name=hss_daemon --chdir=' + self.SourceCodePath + '/scripts -o ' + self.SourceCodePath + '/scripts/hss_' + self.testCase_id + '.log ./my-hss.sh', '\$', 5) elif re.match('OAI', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC HSS') mySSH.command('cd ' + self.SourceCodePath, '\$', 5) mySSH.command('source oaienv', '\$', 5) mySSH.command('cd scripts', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./run_hss 2>&1 | stdbuf -o0 awk \'{ print strftime("[%Y/%m/%d %H:%M:%S] ",systime()) $0 }\' | stdbuf -o0 tee -a hss_' + self.testCase_id + '.log &', 'Core state: 2 -> 3', 35) elif re.match('ltebox', self.Type, re.IGNORECASE): logging.debug('Using the ltebox simulated HSS') mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('cd /opt/hss_sim0609', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S rm -f hss.log', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S echo "Starting sudo session" && sudo su -c "screen -dm -S simulated_hss ./starthss"', '\$', 5) else: logging.error('This option should not occur!') mySSH.close() if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
def InitializeSPGW(self): if self.IPAddress == '' or self.UserName == '' or self.Password == '' or self.SourceCodePath == '' or self.Type == '': HELP.GenericHelp(CONST.Version) HELP.EPCSrvHelp(self.IPAddress, self.UserName, self.Password, self.SourceCodePath, self.Type) sys.exit('Insufficient EPC Parameters') mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) if re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 SPGW-CUPS in Docker') mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup tshark -i eth0 -i lo:p5c -i lo:s5c -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwu-tiny /bin/bash -c "nohup tshark -i eth0 -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 5) time.sleep(5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwc /bin/bash -c "nohup ./bin/oai_spgwc -o -c ./etc/spgw_c.conf > spgwc_check_run.log 2>&1"', '\$', 5) time.sleep(5) mySSH.command('docker exec -d ' + self.containerPrefix + '-oai-spgwu-tiny /bin/bash -c "nohup ./bin/oai_spgwu -o -c ./etc/spgw_u.conf > spgwu_check_run.log 2>&1"', '\$', 5) elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE): logging.debug('Using the OAI EPC Release 14 SPGW-CUPS') mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S rm -f spgwc_' + self.testCase_id + '.log spgwu_' + self.testCase_id + '.log', '\$', 5) mySSH.command('echo "spgwc -c /usr/local/etc/oai/spgw_c.conf" > ./my-spgwc.sh', '\$', 5) mySSH.command('chmod 755 ./my-spgwc.sh', '\$', 5) mySSH.command('sudo daemon --unsafe --name=spgwc_daemon --chdir=' + self.SourceCodePath + '/scripts -o ' + self.SourceCodePath + '/scripts/spgwc_' + self.testCase_id + '.log ./my-spgwc.sh', '\$', 5) time.sleep(5) mySSH.command('echo "spgwu -c /usr/local/etc/oai/spgw_u.conf" > ./my-spgwu.sh', '\$', 5) mySSH.command('chmod 755 ./my-spgwu.sh', '\$', 5) mySSH.command('sudo daemon --unsafe --name=spgwu_daemon --chdir=' + self.SourceCodePath + '/scripts -o ' + self.SourceCodePath + '/scripts/spgwu_' + self.testCase_id + '.log ./my-spgwu.sh', '\$', 5) elif re.match('OAI', self.Type, re.IGNORECASE): mySSH.command('cd ' + self.SourceCodePath, '\$', 5) mySSH.command('source oaienv', '\$', 5) mySSH.command('cd scripts', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./run_spgw 2>&1 | stdbuf -o0 tee -a spgw_' + self.testCase_id + '.log &', 'Initializing SPGW-APP task interface: DONE', 30) elif re.match('ltebox', self.Type, re.IGNORECASE): mySSH.command('cd /opt/ltebox/tools', '\$', 5) mySSH.command('echo ' + self.Password + ' | sudo -S ./start_xGw', '\$', 5) else: logging.error('This option should not occur!') mySSH.close() if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
#----------------------------------------------------------- # COTS UE instanciation #----------------------------------------------------------- #COTS_UE instanciation and ADB server init #ue id and ue mode are retrieved from xml COTS_UE = cls_cots_ue.CotsUe(CiTestObj.ADBIPAddress, CiTestObj.ADBUserName, CiTestObj.ADBPassword) #----------------------------------------------------------- # mode amd XML class (action) analysis #----------------------------------------------------------- cwd = os.getcwd() if re.match('^TerminateeNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') RAN.eNB_instance = 0 RAN.eNB_serverId[0] = '0' RAN.eNBSourceCodePath = '/tmp/' RAN.TerminateeNB(HTML, EPC) elif re.match('^TerminateUE$', mode, re.IGNORECASE): if (CiTestObj.ADBIPAddress == '' or CiTestObj.ADBUserName == '' or CiTestObj.ADBPassword == ''): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) CiTestObj.TerminateUE(HTML, COTS_UE) elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): if CiTestObj.UEIPAddress == '' or CiTestObj.UEUserName == '' or CiTestObj.UEPassword == '': HELP.GenericHelp(CONST.Version)
def DeployEpc(self, HTML): logging.debug('Trying to deploy') if not re.match('OAI-Rel14-Docker', self.Type, re.IGNORECASE): HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER) HTML.CreateHtmlTabFooter(False) sys.exit('Deploy not possible with this EPC type: ' + self.Type) if self.IPAddress == '' or self.UserName == '' or self.Password == '' or self.SourceCodePath == '' or self.Type == '': HELP.GenericHelp(CONST.Version) HELP.EPCSrvHelp(self.IPAddress, self.UserName, self.Password, self.SourceCodePath, self.Type) sys.exit('Insufficient EPC Parameters') mySSH = SSH.SSHConnection() mySSH.open(self.IPAddress, self.UserName, self.Password) mySSH.command('docker-compose --version', '\$', 5) result = re.search('docker-compose version 1', mySSH.getBefore()) if result is None: mySSH.close() HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER) HTML.CreateHtmlTabFooter(False) sys.exit('docker-compose not installed on ' + self.IPAddress) mySSH.command( 'if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5) mySSH.command( 'if [ -d ' + self.SourceCodePath + '/logs ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/logs ; fi', '\$', 5) mySSH.command( 'mkdir -p ' + self.SourceCodePath + '/scripts ' + self.SourceCodePath + '/logs', '\$', 5) # deploying and configuring the cassandra database # container names and services are currently hard-coded. # they could be recovered by: # - docker-compose config --services # - docker-compose config | grep container_name mySSH.command('cd ' + self.SourceCodePath + '/scripts', '\$', 5) mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/docker-compose.yml', self.SourceCodePath + '/scripts') mySSH.command( 'wget --quiet --tries=3 --retry-connrefused https://raw.githubusercontent.com/OPENAIRINTERFACE/openair-hss/develop/src/hss_rel14/db/oai_db.cql', '\$', 30) mySSH.command('docker-compose down', '\$', 60) mySSH.command('docker-compose up -d db_init', '\$', 60) # databases take time... time.sleep(10) cnt = 0 db_init_status = False while (cnt < 10): mySSH.command('docker logs prod-db-init', '\$', 5) result = re.search('OK', mySSH.getBefore()) if result is not None: cnt = 10 db_init_status = True else: time.sleep(5) cnt += 1 mySSH.command('docker rm -f prod-db-init', '\$', 5) if not db_init_status: HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER) HTML.CreateHtmlTabFooter(False) sys.exit('Cassandra DB deployment/configuration went wrong!') # deploying EPC cNFs mySSH.command('docker-compose up -d oai_spgwu', '\$', 60) listOfContainers = 'prod-cassandra prod-oai-hss prod-oai-mme prod-oai-spgwc prod-oai-spgwu-tiny' expectedHealthyContainers = 5 # Checking for additional services mySSH.command('docker-compose config', '\$', 5) configResponse = mySSH.getBefore() if configResponse.count('flexran_rtc') == 1: mySSH.command('docker-compose up -d flexran_rtc', '\$', 60) listOfContainers += ' prod-flexran-rtc' expectedHealthyContainers += 1 if configResponse.count('trf_gen') == 1: mySSH.command('docker-compose up -d trf_gen', '\$', 60) listOfContainers += ' prod-trf-gen' expectedHealthyContainers += 1 # Checking if all are healthy cnt = 0 while (cnt < 3): mySSH.command( 'docker inspect --format=\'{{.State.Health.Status}}\' ' + listOfContainers, '\$', 10) unhealthyNb = mySSH.getBefore().count('unhealthy') healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb startingNb = mySSH.getBefore().count('starting') if healthyNb == expectedHealthyContainers: cnt = 10 else: time.sleep(10) cnt += 1 logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') if healthyNb == expectedHealthyContainers: mySSH.command( 'docker exec -d prod-oai-hss /bin/bash -c "nohup tshark -i any -f \'port 9042 or port 3868\' -w /tmp/hss_check_run.pcap 2>&1 > /dev/null"', '\$', 5) mySSH.command( 'docker exec -d prod-oai-mme /bin/bash -c "nohup tshark -i any -f \'port 3868 or port 2123 or port 36412\' -w /tmp/mme_check_run.pcap 2>&1 > /dev/null"', '\$', 10) mySSH.command( 'docker exec -d prod-oai-spgwc /bin/bash -c "nohup tshark -i any -f \'port 2123 or port 8805\' -w /tmp/spgwc_check_run.pcap 2>&1 > /dev/null"', '\$', 10) # on SPGW-U, not capturing on SGI to avoid huge file mySSH.command( 'docker exec -d prod-oai-spgwu-tiny /bin/bash -c "nohup tshark -i any -f \'port 8805\' -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 10) mySSH.close() logging.debug('Deployment OK') HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK) else: mySSH.close() logging.debug('Deployment went wrong') HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER)
def BuildImage(self, HTML): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId[self.eNB_instance] == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId[self.eNB_instance] == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') logging.debug('Building on server: ' + lIpAddr) mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) # Checking the hostname to get adapted on cli and dockerfileprefixes mySSH.command('hostnamectl', '\$', 5) result = re.search('Ubuntu|Red Hat', mySSH.getBefore()) self.host = result.group(0) if self.host == 'Ubuntu': self.cli = 'docker' self.dockerfileprefix = '.ubuntu18' elif self.host == 'Red Hat': self.cli = 'podman' self.dockerfileprefix = '.rhel8.2' imageNames = [] result = re.search('eNB', self.imageKind) # Creating a tupple with the imageName and the DockerFile prefix pattern on obelix if result is not None: imageNames.append(('oai-enb', 'eNB')) else: result = re.search('gNB', self.imageKind) if result is not None: imageNames.append(('oai-gnb', 'gNB')) else: result = re.search('all', self.imageKind) if result is not None: imageNames.append(('oai-enb', 'eNB')) imageNames.append(('oai-gnb', 'gNB')) imageNames.append(('oai-lte-ue', 'lteUE')) imageNames.append(('oai-nr-ue', 'nrUE')) if len(imageNames) == 0: imageNames.append(('oai-enb', 'eNB')) # Workaround for some servers, we need to erase completely the workspace if self.forcedWorkspaceCleanup: mySSH.command( 'echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) self.testCase_id = HTML.testCase_id # on RedHat/CentOS .git extension is mandatory result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) if result is not None: full_ran_repo_name = self.ranRepository else: full_ran_repo_name = self.ranRepository + '.git' mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) mySSH.command('cd ' + lSourcePath, '\$', 5) mySSH.command( 'if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) # Raphael: here add a check if git clone or git fetch went smoothly mySSH.command('git config user.email "*****@*****.**"', '\$', 5) mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) mySSH.command('mkdir -p cmake_targets/log', '\$', 5) # if the commit ID is provided use it to point to it if self.ranCommitID != '': mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) # if the branch is not develop, then it is a merge request and we need to do # the potential merge. Note that merge conflicts should already been checked earlier imageTag = 'develop' if (self.ranAllowMerge): imageTag = 'ci-temp' if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): mySSH.command( 'git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) mySSH.command( 'git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) # if asterix, copy the entitlement and subscription manager configurations if self.host == 'Red Hat': mySSH.command('mkdir -p tmp/ca/', '\$', 5) mySSH.command('mkdir -p tmp/entitlement/', '\$', 5) mySSH.command('sudo cp /etc/rhsm/ca/redhat-uep.pem tmp/ca/', '\$', 5) mySSH.command( 'sudo cp /etc/pki/entitlement/*.pem tmp/entitlement/', '\$', 5) #mySSH.close() #return 0 sharedimage = 'ran-build' # Let's remove any previous run artifacts if still there mySSH.command(self.cli + ' image prune --force', '\$', 5) mySSH.command(self.cli + ' image rm ' + sharedimage + ':' + imageTag, '\$', 5) for image, pattern in imageNames: mySSH.command(self.cli + ' image rm ' + image + ':' + imageTag, '\$', 5) # Build the shared image mySSH.command( self.cli + ' build --target ' + sharedimage + ' --tag ' + sharedimage + ':' + imageTag + ' --file docker/Dockerfile.ran' + self.dockerfileprefix + ' --build-arg NEEDED_GIT_PROXY="http://proxy.eurecom.fr:8080" . > cmake_targets/log/ran-build.log 2>&1', '\$', 1600) # Build the target image(s) previousImage = sharedimage + ':' + imageTag danglingShaOnes = [] for image, pattern in imageNames: # the archived Dockerfiles have "ran-build:latest" as base image # we need to update them with proper tag mySSH.command( 'sed -i -e "s#' + sharedimage + ':latest#' + sharedimage + ':' + imageTag + '#" docker/Dockerfile.' + pattern + self.dockerfileprefix, '\$', 5) mySSH.command( self.cli + ' build --target ' + image + ' --tag ' + image + ':' + imageTag + ' --file docker/Dockerfile.' + pattern + self.dockerfileprefix + ' . > cmake_targets/log/' + image + '.log 2>&1', '\$', 1200) # Retrieving the dangling image(s) for the log collection mySSH.command( self.cli + ' images --filter "dangling=true" --filter "since=' + previousImage + '" -q | sed -e "s#^#sha=#"', '\$', 5) result = re.search('sha=(?P<imageShaOne>[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) if result is not None: danglingShaOnes.append((image, result.group('imageShaOne'))) previousImage = image + ':' + imageTag imageTag = 'ci-temp' # First verify if images were properly created. status = True mySSH.command( self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + sharedimage + ':' + imageTag, '\$', 5) if mySSH.getBefore().count('No such object') != 0: logging.error('Could not build properly ran-build') status = False else: result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) if result is not None: imageSize = float(result.group('size')) imageSize = imageSize / 1000 if imageSize < 1000: logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') self.allImagesSize['ran-build'] = str(round(imageSize, 1)) + ' kbytes' else: imageSize = imageSize / 1000 if imageSize < 1000: logging.debug('\u001B[1m ran-build size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') self.allImagesSize['ran-build'] = str( round(imageSize, 1)) + ' Mbytes' else: imageSize = imageSize / 1000 logging.debug('\u001B[1m ran-build size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') self.allImagesSize['ran-build'] = str( round(imageSize, 1)) + ' Gbytes' else: logging.debug('ran-build size is unknown') for image, pattern in imageNames: mySSH.command( self.cli + ' image inspect --format=\'Size = {{.Size}} bytes\' ' + image + ':' + imageTag, '\$', 5) if mySSH.getBefore().count('No such object') != 0: logging.error('Could not build properly ' + image) status = False else: result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', mySSH.getBefore()) if result is not None: imageSize = float(result.group('size')) imageSize = imageSize / 1000 if imageSize < 1000: logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' kbytes\u001B[0m') self.allImagesSize[image] = str(round(imageSize, 1)) + ' kbytes' else: imageSize = imageSize / 1000 if imageSize < 1000: logging.debug('\u001B[1m ' + image + ' size is ' + ('%.0f' % imageSize) + ' Mbytes\u001B[0m') self.allImagesSize[image] = str(round( imageSize, 1)) + ' Mbytes' else: imageSize = imageSize / 1000 logging.debug('\u001B[1m ' + image + ' size is ' + ('%.3f' % imageSize) + ' Gbytes\u001B[0m') self.allImagesSize[image] = str(round( imageSize, 1)) + ' Gbytes' else: logging.debug('ran-build size is unknown') if not status: mySSH.close() logging.error('\u001B[1m Building OAI Images Failed\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'KO', CONST.ALL_PROCESSES_OK) #HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize) HTML.CreateHtmlTabFooter(False) sys.exit(1) # Recover build logs, for the moment only possible when build is successful mySSH.command( self.cli + ' create --name test ' + sharedimage + ':' + imageTag, '\$', 5) mySSH.command('mkdir -p cmake_targets/log/ran-build', '\$', 5) mySSH.command( self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-build', '\$', 5) mySSH.command(self.cli + ' rm -f test', '\$', 5) for image, shaone in danglingShaOnes: mySSH.command('mkdir -p cmake_targets/log/' + image, '\$', 5) mySSH.command(self.cli + ' create --name test ' + shaone, '\$', 5) mySSH.command( self.cli + ' cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/' + image, '\$', 5) mySSH.command(self.cli + ' rm -f test', '\$', 5) mySSH.command(self.cli + ' image prune --force', '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5) mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5) #mySSH.close() mySSH.command('cd /tmp/CI-eNB/cmake_targets', '\$', 5) if (os.path.isfile('./build_log_' + self.testCase_id + '.zip')): os.remove('./build_log_' + self.testCase_id + '.zip') mySSH.command( 'zip -r -qq build_log_' + self.testCase_id + '.zip build_log_' + self.testCase_id, '\$', 5) mySSH.copyin( lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '.zip', '.') #mySSH.command('rm -f build_log_' + self.testCase_id + '.zip','\$', 5) mySSH.close() ZipFile('build_log_' + self.testCase_id + '.zip').extractall('.') #Trying to identify the errors and warnings for each built images imageNames1 = imageNames shared = ('ran-build', 'ran') imageNames1.insert(0, shared) for image, pattern in imageNames1: files = {} file_list = [ f for f in os.listdir('build_log_' + self.testCase_id + '/' + image) if os.path.isfile( os.path.join('build_log_' + self.testCase_id + '/' + image, f)) and f.endswith('.txt') ] for fil in file_list: errorandwarnings = {} warningsNo = 0 errorsNo = 0 with open('build_log_{}/{}/{}'.format(self.testCase_id, image, fil), mode='r') as inputfile: for line in inputfile: result = re.search(' ERROR ', str(line)) if result is not None: errorsNo += 1 result = re.search(' error:', str(line)) if result is not None: errorsNo += 1 result = re.search(' WARNING ', str(line)) if result is not None: warningsNo += 1 result = re.search(' warning:', str(line)) if result is not None: warningsNo += 1 errorandwarnings['errors'] = errorsNo errorandwarnings['warnings'] = warningsNo errorandwarnings['status'] = status files[fil] = errorandwarnings self.collectInfo[image] = files logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m') HTML.CreateHtmlTestRow(self.imageKind, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlNextTabHeaderTestRow(self.collectInfo, self.allImagesSize)
def UndeployObject(self, HTML, RAN): logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m') if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId[self.eNB_instance] == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId[self.eNB_instance] == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command( 'cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) # Currently support only one mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) result = re.search( 'container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) if result is not None: containerName = result.group('container_name') mySSH.command('docker kill --signal INT ' + containerName, '\$', 30) time.sleep(5) mySSH.command( 'docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30) mySSH.command('docker rm -f ' + containerName, '\$', 30) # Putting the CPUs back in a idle state, we do that only on a few servers mySSH.command('hostname', '\$', 5) result = re.search('obelix|asterix', mySSH.getBefore()) if result is not None: mySSH.command( 'if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -E; fi', '\$', 5) mySSH.close() # Analyzing log file! copyin_res = mySSH.copyin( lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '.') nodeB_prefix = 'e' if (copyin_res == -1): HTML.htmleNBFailureMsg = 'Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) else: logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + self.eNB_logFile[self.eNB_instance]) logStatus = RAN.AnalyzeLogFile_eNB( self.eNB_logFile[self.eNB_instance], HTML) if (logStatus < 0): HTML.CreateHtmlTestRow(RAN.runtime_stats, 'KO', logStatus) else: HTML.CreateHtmlTestRow(RAN.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
def DeployObject(self, HTML, EPC): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId[self.eNB_instance] == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId[self.eNB_instance] == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m') mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) # Putting the CPUs in a good state, we do that only on a few servers mySSH.command('hostname', '\$', 5) result = re.search('obelix|asterix', mySSH.getBefore()) if result is not None: mySSH.command( 'if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -D 0; fi', '\$', 5) time.sleep(5) mySSH.command( 'cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5) imageTag = 'develop' if (self.ranAllowMerge): imageTag = 'ci-temp' mySSH.command( 'sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) localMmeIpAddr = EPC.MmeIPAddress mySSH.command( 'sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2) if self.flexranCtrlDeployed: mySSH.command( 'sed -i -e \'s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: "yes"/\' ci-docker-compose.yml', '\$', 2) mySSH.command( 'sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2) else: mySSH.command( 'sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \"no\"/" ci-docker-compose.yml', '\$', 2) mySSH.command( 'sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2) # Currently support only one mySSH.command( 'docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@"', '\$', 2) result = re.search('service=(?P<svc_name>[a-zA-Z0-9\_]+)', mySSH.getBefore()) if result is not None: svcName = result.group('svc_name') mySSH.command( 'docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 2) # Checking Status mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5) result = re.search( 'container_name: (?P<container_name>[a-zA-Z0-9\-\_]+)', mySSH.getBefore()) unhealthyNb = 0 healthyNb = 0 startingNb = 0 containerName = '' if result is not None: containerName = result.group('container_name') time.sleep(5) cnt = 0 while (cnt < 3): mySSH.command( 'docker inspect --format=\'{{.State.Health.Status}}\' ' + containerName, '\$', 5) unhealthyNb = mySSH.getBefore().count('unhealthy') healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb startingNb = mySSH.getBefore().count('starting') if healthyNb == 1: cnt = 10 else: time.sleep(10) cnt += 1 logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') status = False if healthyNb == 1: cnt = 0 while (cnt < 20): mySSH.command( 'docker logs ' + containerName + ' | egrep --text --color=never -i "wait|sync|Starting"', '\$', 30) result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore()) if result is None: time.sleep(6) cnt += 1 else: cnt = 100 status = True logging.info( '\u001B[1m Deploying OAI object Pass\u001B[0m') time.sleep(10) mySSH.close() self.testCase_id = HTML.testCase_id self.eNB_logFile[ self.eNB_instance] = 'enb_' + self.testCase_id + '.log' if status: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) else: HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK)
def BuildeNB(self): if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') if self.eNB_serverId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) # Check if we build an 5G-NR gNB or an LTE eNB or an OCP eNB result = re.search('--eNBocp', self.Build_eNB_args) if result is not None: self.air_interface[self.eNB_instance] = 'ocp-enb' else: result = re.search('--gNB', self.Build_eNB_args) if result is not None: self.air_interface[self.eNB_instance] = 'nr-softmodem' else: self.air_interface[self.eNB_instance] = 'lte-softmodem' # Worakround for some servers, we need to erase completely the workspace if self.Build_eNB_forced_workspace_cleanup: mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf ' + lSourcePath, '\$', 15) if self.htmlObj is not None: self.testCase_id = self.htmlObj.testCase_id else: self.testCase_id = '000000' # on RedHat/CentOS .git extension is mandatory result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository) if result is not None: full_ran_repo_name = self.ranRepository else: full_ran_repo_name = self.ranRepository + '.git' mySSH.command('mkdir -p ' + lSourcePath, '\$', 5) mySSH.command('cd ' + lSourcePath, '\$', 5) mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600) # Raphael: here add a check if git clone or git fetch went smoothly mySSH.command('git config user.email "*****@*****.**"', '\$', 5) mySSH.command('git config user.name "OAI Jenkins"', '\$', 5) # Checking the BUILD INFO file if not self.backgroundBuild: mySSH.command('ls *.txt', '\$', 5) result = re.search('LAST_BUILD_INFO', mySSH.getBefore()) if result is not None: mismatch = False mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2) result = re.search(self.ranCommitID, mySSH.getBefore()) if result is None: mismatch = True mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) if (self.ranAllowMerge): result = re.search('YES', mySSH.getBefore()) if result is None: mismatch = True mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2) if self.ranTargetBranch == '': result = re.search('develop', mySSH.getBefore()) else: result = re.search(self.ranTargetBranch, mySSH.getBefore()) if result is None: mismatch = True else: result = re.search('NO', mySSH.getBefore()) if result is None: mismatch = True if not mismatch: mySSH.close() if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK) return mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30) # if the commit ID is provided use it to point to it if self.ranCommitID != '': mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 5) # if the branch is not develop, then it is a merge request and we need to do # the potential merge. Note that merge conflicts should already been checked earlier if (self.ranAllowMerge): if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) mySSH.command('source oaienv', '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) mySSH.command('mkdir -p log', '\$', 5) mySSH.command('chmod 777 log', '\$', 5) # no need to remove in log (git clean did the trick) if self.backgroundBuild: mySSH.command('echo "./build_oai ' + self.Build_eNB_args + '" > ./my-lte-softmodem-build.sh', '\$', 5) mySSH.command('chmod 775 ./my-lte-softmodem-build.sh', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S ls', '\$', 5) mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-build.sh' + ' > ' + lSourcePath + '/cmake_targets/compile_oai_enb.log ' + ' 2>&1 &', lUserName, 5) mySSH.close() if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.Build_eNB_args, 'OK', CONST.ALL_PROCESSES_OK) self.backgroundBuildTestId[int(self.eNB_instance)] = self.testCase_id return mySSH.command('stdbuf -o0 ./build_oai ' + self.Build_eNB_args + ' 2>&1 | stdbuf -o0 tee compile_oai_enb.log', 'Bypassing the Tests|build have failed', 1500) mySSH.close() self.checkBuildeNB(lIpAddr, lUserName, lPassWord, lSourcePath, self.testCase_id)
def TerminateeNB(self): if self.eNB_serverId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5) if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): nodeB_prefix = 'e' else: nodeB_prefix = 'g' mySSH.command('stdbuf -o0 ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5) result = re.search('(-softmodem|ocp)', mySSH.getBefore()) if result is not None: mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGINT -r .*-softmodem ocp-enb || true', '\$', 5) time.sleep(10) mySSH.command('stdbuf -o0 ps -aux | grep --color=never -e softmodem -e ocp-enb | grep -v grep', '\$', 5) result = re.search('(-softmodem|ocp)', mySSH.getBefore()) if result is not None: mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5) time.sleep(5) mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) mySSH.close() # If tracer options is on, stopping tshark on EPC side result = re.search('T_stdout', str(self.Initialize_eNB_args)) if (result is not None) and (self.epcObj is not None): localEpcIpAddr = self.epcObj.IPAddress localEpcUserName = self.epcObj.UserName localEpcPassword = self.epcObj.Password mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword) logging.debug('\u001B[1m Stopping tshark \u001B[0m') mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) time.sleep(1) if self.epcPcapFile != '': mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5) mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.') mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.') mySSH.close() logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5) enbLogFile = self.eNBLogFiles[int(self.eNB_instance)] raw_record_file = enbLogFile.replace('.log', '_record.raw') replay_log_file = enbLogFile.replace('.log', '_replay.log') extracted_txt_file = enbLogFile.replace('.log', '_extracted_messages.txt') extracted_log_file = enbLogFile.replace('.log', '_extracted_messages.log') mySSH.command('./extract_config -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + extracted_txt_file, '\$', 5) mySSH.command('echo $USER; nohup ./replay -i ' + lSourcePath + '/cmake_targets/' + raw_record_file + ' > ' + lSourcePath + '/cmake_targets/' + replay_log_file + ' 2>&1 &', lUserName, 5) mySSH.command('./textlog -d ' + lSourcePath + '/cmake_targets/' + extracted_txt_file + ' -no-gui -ON -full > ' + lSourcePath + '/cmake_targets/' + extracted_log_file, '\$', 5) mySSH.close() mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + extracted_log_file, '.') logging.debug('\u001B[1m Analyzing eNB replay logfile \u001B[0m') logStatus = self.AnalyzeLogFile_eNB(extracted_log_file) if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) self.eNBLogFiles[int(self.eNB_instance)] = '' else: analyzeFile = False if self.eNBLogFiles[int(self.eNB_instance)] != '': analyzeFile = True fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)] self.eNBLogFiles[int(self.eNB_instance)] = '' if analyzeFile: copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.') if (copyin_res == -1): logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m') if self.htmlObj is not None: self.htmlObj.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!' self.htmlObj.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE) self.eNBmbmsEnables[int(self.eNB_instance)] = False return if self.eNB_serverId != '0': mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/') logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze) logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze) if (logStatus < 0): if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow('N/A', 'KO', logStatus) self.preamtureExit = True self.eNBmbmsEnables[int(self.eNB_instance)] = False return else: if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) else: if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) self.eNBmbmsEnables[int(self.eNB_instance)] = False self.eNBstatuses[int(self.eNB_instance)] = -1
def InitializeeNB(self): if self.eNB_serverId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword lSourcePath = self.eNBSourceCodePath elif self.eNB_serverId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password lSourcePath = self.eNB1SourceCodePath elif self.eNB_serverId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password lSourcePath = self.eNB2SourceCodePath if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') if self.htmlObj is not None: self.testCase_id = self.htmlObj.testCase_id else: self.testCase_id = '000000' mySSH = SSH.SSHConnection() if (self.pStatus < 0): if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus) self.htmlObj.CreateHtmlTabFooter(False) sys.exit(1) # If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB result = re.search('T_stdout', str(self.Initialize_eNB_args)) if (result is not None) and (self.epcObj is not None): localEpcIpAddr = self.epcObj.IPAddress localEpcUserName = self.epcObj.UserName localEpcPassword = self.epcObj.Password mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword) mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) result = re.search('interfaceToUse=(?P<eth_interface>[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) if result is not None: eth_interface = result.group('eth_interface') logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') self.epcPcapFile = 'enb_' + self.testCase_id + '_s1log.pcap' mySSH.command('echo ' + localEpcPassword + ' | sudo -S rm -f /tmp/' + self.epcPcapFile , '\$', 5) mySSH.command('echo $USER; nohup sudo tshark -f "host ' + lIpAddr +'" -i ' + eth_interface + ' -w /tmp/' + self.epcPcapFile + ' > /tmp/tshark.log 2>&1 &', localEpcUserName, 5) mySSH.close() mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('cd ' + lSourcePath, '\$', 5) # Initialize_eNB_args usually start with -O and followed by the location in repository full_config_file = self.Initialize_eNB_args.replace('-O ','') extra_options = '' extIdx = full_config_file.find('.conf') if (extIdx > 0): extra_options = full_config_file[extIdx + 5:] # if tracer options is on, compiling and running T Tracer result = re.search('T_stdout', str(extra_options)) if result is not None: logging.debug('\u001B[1m Compiling and launching T Tracer\u001B[0m') mySSH.command('cd common/utils/T/tracer', '\$', 5) mySSH.command('make', '\$', 10) mySSH.command('echo $USER; nohup ./record -d ../T_messages.txt -o ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.raw -ON -off VCD -off HEAVY -off LEGACY_GROUP_TRACE -off LEGACY_GROUP_DEBUG > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '_record.log 2>&1 &', lUserName, 5) mySSH.command('cd ' + lSourcePath, '\$', 5) full_config_file = full_config_file[:extIdx + 5] config_path, config_file = os.path.split(full_config_file) else: sys.exit('Insufficient Parameter') ci_full_config_file = config_path + '/ci-' + config_file rruCheck = False result = re.search('^rru|^rcc|^du.band', str(config_file)) if result is not None: rruCheck = True # do not reset board twice in IF4.5 case result = re.search('^rru|^enb|^du.band', str(config_file)) if result is not None: mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60) result = re.search('type: b200', mySSH.getBefore()) if result is not None: logging.debug('Found a B2xx device --> resetting it') mySSH.command('echo ' + lPassWord + ' | sudo -S b2xx_fx3_utils --reset-device', '\$', 10) # Reloading FGPA bin firmware mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 60) # Make a copy and adapt to EPC / eNB IP addresses mySSH.command('cp ' + full_config_file + ' ' + ci_full_config_file, '\$', 5) if self.epcObj is not None: localMmeIpAddr = self.epcObj.MmeIPAddress mySSH.command('sed -i -e \'s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/\' ' + ci_full_config_file, '\$', 2); mySSH.command('sed -i -e \'s/CI_ENB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2); mySSH.command('sed -i -e \'s/CI_GNB_IP_ADDR/' + lIpAddr + '/\' ' + ci_full_config_file, '\$', 2); mySSH.command('sed -i -e \'s/CI_RCC_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2); mySSH.command('sed -i -e \'s/CI_RRU1_IP_ADDR/' + self.eNB1IPAddress + '/\' ' + ci_full_config_file, '\$', 2); mySSH.command('sed -i -e \'s/CI_RRU2_IP_ADDR/' + self.eNB2IPAddress + '/\' ' + ci_full_config_file, '\$', 2); mySSH.command('sed -i -e \'s/CI_FR1_CTL_ENB_IP_ADDR/' + self.eNBIPAddress + '/\' ' + ci_full_config_file, '\$', 2); if self.flexranCtrlInstalled and self.flexranCtrlStarted: mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED = "yes";/\' ' + ci_full_config_file, '\$', 2); else: mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED = "no";/\' ' + ci_full_config_file, '\$', 2); self.eNBmbmsEnables[int(self.eNB_instance)] = False mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2); result = re.search('yes', mySSH.getBefore()) if result is not None: self.eNBmbmsEnables[int(self.eNB_instance)] = True logging.debug('\u001B[1m MBMS is enabled on this eNB\u001B[0m') result = re.search('noS1', str(self.Initialize_eNB_args)) eNBinNoS1 = False if result is not None: eNBinNoS1 = True logging.debug('\u001B[1m eNB is in noS1 configuration \u001B[0m') # Launch eNB with the modified config file mySSH.command('source oaienv', '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) if self.air_interface == 'nr': mySSH.command('if [ -e rbconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm rbconfig.raw; fi', '\$', 5) mySSH.command('if [ -e reconfig.raw ]; then echo ' + lPassWord + ' | sudo -S rm reconfig.raw; fi', '\$', 5) # NOTE: WE SHALL do a check if the executable is present (in case build went wrong) mySSH.command('echo "ulimit -c unlimited && ./ran_build/build/' + self.air_interface[self.eNB_instance] + ' -O ' + lSourcePath + '/' + ci_full_config_file + extra_options + '" > ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) mySSH.command('chmod 775 ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) mySSH.command('echo ' + lPassWord + ' | sudo -S rm -Rf enb_' + self.testCase_id + '.log', '\$', 5) mySSH.command('echo $USER; nohup sudo -E ./my-lte-softmodem-run' + str(self.eNB_instance) + '.sh > ' + lSourcePath + '/cmake_targets/enb_' + self.testCase_id + '.log 2>&1 &', lUserName, 10) self.eNBLogFiles[int(self.eNB_instance)] = 'enb_' + self.testCase_id + '.log' if extra_options != '': self.eNBOptions[int(self.eNB_instance)] = extra_options time.sleep(6) doLoop = True loopCounter = 20 enbDidSync = False while (doLoop): loopCounter = loopCounter - 1 if (loopCounter == 0): # In case of T tracer recording, we may need to kill it result = re.search('T_stdout', str(self.Initialize_eNB_args)) if result is not None: mySSH.command('killall --signal SIGKILL record', '\$', 5) mySSH.close() doLoop = False logging.error('\u001B[1;37;41m eNB/gNB/ocp-eNB logging system did not show got sync! \u001B[0m') if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'KO', CONST.ALL_PROCESSES_OK) # In case of T tracer recording, we need to kill tshark on EPC side result = re.search('T_stdout', str(self.Initialize_eNB_args)) if (result is not None) and (self.epcObj is not None): localEpcIpAddr = self.epcObj.IPAddress localEpcUserName = self.epcObj.UserName localEpcPassword = self.epcObj.Password mySSH.open(localEpcIpAddr, localEpcUserName, localEpcPassword) logging.debug('\u001B[1m Stopping tshark \u001B[0m') mySSH.command('echo ' + localEpcPassword + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) if self.epcPcapFile != '': time.sleep(0.5) mySSH.command('echo ' + localEpcPassword + ' | sudo -S chmod 666 /tmp/' + self.epcPcapFile, '\$', 5) mySSH.close() time.sleep(1) if self.epcPcapFile != '': copyin_res = mySSH.copyin(localEpcIpAddr, localEpcUserName, localEpcPassword, '/tmp/' + self.epcPcapFile, '.') if (copyin_res == 0): mySSH.copyout(lIpAddr, lUserName, lPassWord, self.epcPcapFile, lSourcePath + '/cmake_targets/.') self.prematureExit = True return else: mySSH.command('stdbuf -o0 cat enb_' + self.testCase_id + '.log | egrep --text --color=never -i "wait|sync|Starting"', '\$', 4) if rruCheck: result = re.search('wait RUs', mySSH.getBefore()) else: result = re.search('got sync|Starting F1AP at CU', mySSH.getBefore()) if result is None: time.sleep(6) else: doLoop = False enbDidSync = True time.sleep(10) rruCheck = False result = re.search('^rru|^du.band', str(config_file)) if result is not None: rruCheck = True if enbDidSync and eNBinNoS1 and not rruCheck: mySSH.command('ifconfig oaitun_enb1', '\$', 4) mySSH.command('ifconfig oaitun_enb1', '\$', 4) result = re.search('inet addr:1|inet 1', mySSH.getBefore()) if result is not None: logging.debug('\u001B[1m oaitun_enb1 interface is mounted and configured\u001B[0m') else: logging.error('\u001B[1m oaitun_enb1 interface is either NOT mounted or NOT configured\u001B[0m') if self.eNBmbmsEnables[int(self.eNB_instance)]: mySSH.command('ifconfig oaitun_enm1', '\$', 4) result = re.search('inet addr', mySSH.getBefore()) if result is not None: logging.debug('\u001B[1m oaitun_enm1 interface is mounted and configured\u001B[0m') else: logging.error('\u001B[1m oaitun_enm1 interface is either NOT mounted or NOT configured\u001B[0m') if enbDidSync: self.eNBstatuses[int(self.eNB_instance)] = int(self.eNB_serverId) mySSH.close() if self.htmlObj is not None: self.htmlObj.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' -O ' + config_file + extra_options, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('\u001B[1m Initialize eNB/gNB/ocp-eNB Completed\u001B[0m')