def showScene(self, sceneId, displayId=0, blockUntilShown=True, timeout=DEFAULT_WAIT_FOR_MESSAGE_TIMEOUT): ramshCommand = "showSceneOnDisplay -sceneId {0} -displayId {1}".format( sceneId, displayId) if blockUntilShown: confirm_message = "scene_{}_shown".format(sceneId) ramshCommand += " -confirm " + confirm_message log.info( "sending showSceneOnDisplay command for scene {}, waiting for confirmation" .format(sceneId)) success = self.send_ramsh_command( ramshCommand, response_message="confirmation: " + confirm_message, timeout=timeout) if success: log.info( "received confirmation of showSceneOnDisplay command for scene {}" .format(sceneId)) else: self.send_ramsh_command("rinfo") assert success, "Timeout waiting for show event of scene, check rinfo output in renderer log" else: self.send_ramsh_command(ramshCommand, waitForRendererConfirmation=True, timeout=timeout)
def execute_on_target(self, commandToExecute, block=True, env={}, cwd=None, timeout=None): prefix = helper.get_env_var_setting_string(env) command = "{} ".format(prefix) + commandToExecute if cwd: command = "cd {}; ".format(cwd) + command log.info("[{}]{}".format(self.name, command)) stdin, stdout, stderr = self.sshClient.exec_command(command) stdoutBuffer = Buffer() stdoutReader = AsynchronousPipeReader(stdout, stdoutBuffer) stderrBuffer = Buffer() stderrReader = AsynchronousPipeReader(stderr, stderrBuffer) if block: if timeout: # poll on application exit with timeout to prevent infinite blocking endTime = time.time() + timeout while time.time( ) < endTime and not stdout.channel.exit_status_ready(): time.sleep(0.1) if not stdout.channel.exit_status_ready(): return [], ["<timeout>"], 1 # get application exit status blocking. will immediately succeed when timeout was given returnCode = stdout.channel.recv_exit_status() stdoutReader.stop(withTimeout=True) stderrReader.stop(withTimeout=True) return stdoutBuffer.get_all_data(), stderrBuffer.get_all_data( ), returnCode return None
def take_screenshot_and_compare(self, renderer, imageName, testClassName, testRunName, displayNumber, screenshotNumber, percentageOfWrongPixelsAllowed, percentageOfRGBDifferenceAllowedPerPixel, numberOfRequiredUnequalPixels, useSystemCompositorForScreenshot, compareForEquality): log.info("taking screenshot...") referenceImagePath = self._find_reference_image(imageName) refImage = Image.open(referenceImagePath) minWidth = refImage.size[0] minHeight = refImage.size[1] pathToResultScreenshot = self._take_and_transfer_screenshot(renderer, imageName, screenshotNumber, testClassName, testRunName, displayNumber, useSystemCompositorForScreenshot, minWidth, minHeight) if not os.path.isfile(pathToResultScreenshot): log.errorAndAssert("Screenshot not found at {}".format(pathToResultScreenshot)) else: log.info("comparing images...") if (self.percentageOfRGBDifferenceAllowedPerPixelOnTarget > percentageOfRGBDifferenceAllowedPerPixel): log.important_info("Allowing higher difference per pixel because of target value: {}%".format(self.percentageOfRGBDifferenceAllowedPerPixelOnTarget*100)) percentageOfRGBDifferenceAllowedPerPixel = self.percentageOfRGBDifferenceAllowedPerPixelOnTarget if (self.percentageOfWrongPixelsAllowedOnTarget > percentageOfWrongPixelsAllowed): log.important_info("Allowing higher number of wrong pixels because of target value: {}%".format(self.percentageOfWrongPixelsAllowedOnTarget*100)) percentageOfWrongPixelsAllowed = self.percentageOfWrongPixelsAllowedOnTarget helper.compare_images(pathToResultScreenshot, referenceImagePath, percentageOfWrongPixelsAllowed, percentageOfRGBDifferenceAllowedPerPixel, numberOfRequiredUnequalPixels, self.imageDiffScaleFactor, compareForEquality)
def start_application(self, applicationName, args="", binaryDirectoryOnTarget=None, nameExtension="", env={}, dltAppID=None): #ensure binary is there if binaryDirectoryOnTarget: binaryPathOnTarget = binaryDirectoryOnTarget + '/' + applicationName else: binaryPathOnTarget = applicationName (_, _, resultTest) = self.execute_on_target("type " + binaryPathOnTarget) #test -e cannot be used as it does not work for applications in system path if resultTest != 0: log.error("Error: executable '{0}' could not be found (path: '{1}')".format(applicationName, binaryPathOnTarget)) return Application(None, None, None, applicationName, binaryDirectoryOnTarget, nameExtension) prefix = helper.get_env_var_setting_string(self._get_merged_env(env)) #execute application if binaryDirectoryOnTarget: command = "cd {}; {} ./{} {}".format(binaryDirectoryOnTarget, prefix, applicationName, args) else: command = "{} {} {}".format(prefix, applicationName, args) log.info("start_application command: '{}'".format(command)) try: stdin, stdout, stderr = self.sshClient.exec_command(command) except Exception as e: log.error("Error: {0} could not be started (error message: {1})".format(applicationName, e.message)) return Application(None, None, None, applicationName, binaryDirectoryOnTarget, nameExtension) application = Application(stdin, stdout, stderr, applicationName, binaryDirectoryOnTarget, nameExtension) application.started = True return application
def _scp(self, source, sourceIsRemote, dest, destIsRemote, dest_has_filename=False): assert not (sourceIsRemote and destIsRemote) assert source.find('*') == -1 assert dest.find('*') == -1 if not dest_has_filename: dest = dest + '/' + os.path.basename(source) try: if sourceIsRemote: log.info("copy remote source '{}' to local dest '{}'".format( source, dest)) self.ftpClient.get(source, dest) if destIsRemote: log.info("copy local source '{}' to remote dest '{}'".format( source, dest)) self.ftpClient.put(source, dest) except Exception as e: print('SCP Error:', e) raise
def execute_on_target(self, commandToExecute, block=True, env={}, cwd=None): prefix = helper.get_env_var_setting_string(env) command = "{} ".format(prefix) + commandToExecute if cwd: command = "cd {}; ".format(cwd) + command log.info("executing '" + command + "' on target") stdin, stdout, stderr = self.sshClient.exec_command(command) stdoutBuffer = Buffer() stdoutReader = AsynchronousPipeReader(stdout, stdoutBuffer) stderrBuffer = Buffer() stderrReader = AsynchronousPipeReader(stderr, stderrBuffer) if (block): returnCode = stdout.channel.recv_exit_status( ) #block till call is finished stdoutReader.stop(withTimeout=True) stderrReader.stop(withTimeout=True) return stdoutBuffer.get_all_data(), stderrBuffer.get_all_data( ), returnCode return None
def setup(self, resultsDir, transfer_binaries): self.fullResultsDirPath = helper.create_result_dir( self.basePath, resultsDir) paramiko.util.log_to_file( os.path.join(self.fullResultsDirPath, 'paramiko.log')) self.createTargets() log.info("turning on all power outlets...") for target in itertools.chain(self.bridgeTargets.itervalues(), self.targets.itervalues()): if target.powerDevice is not None: if not target.powerDevice.switch(target.powerOutletNr, True): log.info( "Could not turn on power outlet for target {0} because the power outlet is not available" .format(target.name)) return False if self.downloadFromCICache: for target in self.targets.itervalues(): downloadSuccessful, version, commitCount = helper.download_tar_from_ci_cache( self.gitCommitHash, target.buildJobName, self.config.ciCacheUrl, self.basePath) if not downloadSuccessful: return False target.ramsesVersion = version target.gitCommitCount = commitCount return self.setupTargets(transfer_binaries and (not self.noTransfer))
def compareUnequal(image1, image2, numberOfRequiredUnequalPixels, percentageOfRGBDifferenceRequiredPerPixel): log.info("Requiring {}% difference on RGBA value per pixel, and require {} really distinct pixels".format(percentageOfRGBDifferenceRequiredPerPixel*100, numberOfRequiredUnequalPixels)) if not _checkImageSizeEqual(image1, image2): return False nrEqualPixels = 0 nrTooSimilarPixels = 0 totalNumberOfPixels = image1.width * image1.height # PIL image comparison is well optimized -> early out if images are identical if image1 == image2: nrEqualPixels = totalNumberOfPixels nrTooSimilarPixels = totalNumberOfPixels else: imageDiff = ImageChops.difference(image1.convert("RGBA"), image2.convert("RGBA")) imageData = imageDiff.getdata() percentageOfRGBDifferenceRequiredPerPixelScaled = int(percentageOfRGBDifferenceRequiredPerPixel*255) for i in range(0, image1.width * image1.height): chMax = max(imageData[i]) if chMax < percentageOfRGBDifferenceRequiredPerPixelScaled: nrTooSimilarPixels += 1 if chMax == 0: nrEqualPixels += 1 log.important_info("Comparison stats: Percentage of too similar pixels: {}% ({})".format(float(nrTooSimilarPixels) / totalNumberOfPixels*100, nrTooSimilarPixels)) log.important_info("Comparison stats: Percentage of exactly equal pixels: {}% ({})".format(float(nrEqualPixels) / totalNumberOfPixels*100, nrEqualPixels)) if totalNumberOfPixels - nrTooSimilarPixels < numberOfRequiredUnequalPixels: log.error("compareUnequal: Not enough unequal pixels, aborting...") return False return True
def _scp(self, source, sourceIsRemote, dest, destIsRemote): fullSource = source fullDest = dest if sourceIsRemote: fullSource = "{0}@{1}:{2}".format(self.username, self.hostname, source.replace(' ', '\ ')) if destIsRemote: fullDest = "{0}@{1}:{2}".format(self.username, self.hostname, dest.replace(' ', '\ ')) log.info("scp source '{}' to dest '{}'".format(fullSource, fullDest)) if self.privateKey is None: return subprocess.call([ "scp", "-P", str(self.sshPort), "-r", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", fullSource, fullDest ]) else: return subprocess.call([ "scp", "-P", str(self.sshPort), "-r", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "IdentityFile={0}".format(self.privateKey), fullSource, fullDest ])
def compareEqual(image1, image2, percentageOfWrongPixelsAllowed, percentageOfRGBDifferenceAllowedPerPixel): log.info("Allowing {}% tolerance on RGBA value per pixel, and {}% wrong pixels".format(percentageOfRGBDifferenceAllowedPerPixel*100, percentageOfWrongPixelsAllowed*100)) if not _checkImageSizeEqual(image1, image2): return False nrWrongPixels = 0 nrDifferentPixels = 0 # PIL image comparison is well optimized -> early out if images are identical if image1 != image2: imageDiff = ImageChops.difference(image1.convert("RGBA"), image2.convert("RGBA")) imageData = imageDiff.getdata() percentageOfRGBDifferenceAllowedPerPixelScaled = int(percentageOfRGBDifferenceAllowedPerPixel*255) for i in range(0, image1.width * image1.height): chMax = max(imageData[i]) if chMax > 0: nrDifferentPixels += 1 if chMax > percentageOfRGBDifferenceAllowedPerPixelScaled: nrWrongPixels += 1 totalNumberOfPixels = image1.width * image1.height log.important_info("Comparison stats: Percentage of wrong pixels: {0}%".format(float(nrWrongPixels) / totalNumberOfPixels*100)) log.important_info("Comparison stats: Percentage of different, but accepted pixels: {0}%".format(float(nrDifferentPixels-nrWrongPixels) / totalNumberOfPixels*100)) if ((float(nrWrongPixels) / totalNumberOfPixels) > percentageOfWrongPixelsAllowed): log.error("compareEqual: Too many wrong pixels, aborting...") return False return True
def _take_and_transfer_screenshot(self, renderer, imageName, screenshotNumber, testClassName, testRunName, displayNumber, useSystemCompositorForScreenshot, minWidth, minHeight): splittedImageName = os.path.splitext(imageName) localScreenshotName = "{}_{}_{}{}".format(splittedImageName[0], screenshotNumber, self.name, splittedImageName[1]) targetScreenshotName = "{}{}_{:04d}_{}".format(self.fixed_screenshot_prefix, self.unique_screenshot_prefix, self.target_screenshot_counter, localScreenshotName) self.target_screenshot_counter += 1 # trigger screenshot(s) if useSystemCompositorForScreenshot: self._take_system_compositor_screenshot(targetScreenshotName, renderer) else: self._take_renderer_screenshot(targetScreenshotName, renderer, displayNumber) resultDirForTest = helper.get_result_dir_subdirectory(self.resultDir, testClassName+"/"+testRunName) self._transfer_screenshots(targetScreenshotName, self.tmpDir, resultDirForTest) localScreenshotFile = os.path.join(resultDirForTest, localScreenshotName) log.info("Store remote {} as local {}".format(targetScreenshotName, localScreenshotFile)) os.rename(os.path.join(resultDirForTest, targetScreenshotName), localScreenshotFile) #check image size pilImage = Image.open(localScreenshotFile) if (pilImage.size[0] < minWidth) or (pilImage.size[1] < minHeight): log.errorAndAssert("Screenshot too small: expected >= {}x{}, got {}x{}".format(minWidth, minHeight, pilImage.size[0], pilImage.size[1])) return localScreenshotFile
def create_diff_images(image1, image2, originalFilePath, scaleFactor): diff = ImageChops.difference(image1, image2) diffScaled = ImageEnhance.Contrast(diff).enhance(scaleFactor) (root, ext) = os.path.splitext(originalFilePath) diff.save(root + "_DIFF" + ext) diffScaled.save(root + "_DIFF_SCALED" + ext) log.info("diff files saved")
def impl_test(self): # check if ltrace tool is available on the present target if not self.target.ltraceCommandSupported: self.skipTest("ltrace command not available") applicationToTrace = "./ramses-local-client-test-{} -tn 3".format( self.target.defaultPlatform) commandToExecute = "ltrace -f -e fopen+fclose+fseek {}".format( applicationToTrace) log.info("Start tracing of " + applicationToTrace) (stdoutdata, stderrdata, returncode) = self.target.execute_on_target( commandToExecute, env=self.target.defaultEnvironment, cwd=self.target.ramsesInstallDir + "/bin") self.assertEquals( returncode, 0, "Tracing failure, Execution of command \"{}\" returned error code {}\nstdout: {}\nstderr: {}" .format(commandToExecute, returncode, stdoutdata, stderrdata)) log.info("Tracing succeeded") hasError = self.hasFileReadingErrors(stderrdata) self.assertEquals( hasError, False, msg= "Found file opening/reading errors. Here is the trace output:\n\n" + "".join(stderrdata))
def create_diff_images(image1, image2, originalFilePath, scaleFactor): diff = ImageChops.difference(image1, image2) diff = diff.convert(mode='RGBA') # following ops cannot work with mode '1' or 'L' that might result from diffing diffScaled = ImageEnhance.Contrast(diff).enhance(scaleFactor) (root, ext) = os.path.splitext(originalFilePath) diff.save(root+"_DIFF"+ext) diffScaled.save(root+"_DIFF_SCALED"+ext) log.info("diff files saved")
def impl_test(self): log.info("making screenshot on target 0") self.renderer.showScene(26) self.validateScreenshotOnTarget(self.renderer, "testClient_threeTriangles.png", self.targets[0]) log.info("making screenshot on target 1") self.renderer2.showScene(26) self.validateScreenshotOnTarget(self.renderer2, "testClient_threeTriangles.png", self.targets[1])
def is_initialised(self, timeout=None): if self.initialisationMessage and not self.initialised and self.initialisationWatchID: log.info("waiting for initialisation message") self.initialised = self.wait_for_msg_in_stdout( self.initialisationWatchID, self.initialisationMessage, timeout) if self.initialised: log.info("initialisation message received") return self.initialised
def compare(x, y, c1, c2, diff): if any(e > 0 for e in diff): nrDifferentPixels[0] += 1 if any(e > percentageOfRGBDifferenceAllowedPerPixel for e in diff): if nrWrongPixels[0] == 0: log.info( "First wrong Pixel {}/{} (source pixel {} / otherPixel {})" .format(x, y, c1, c2)) nrWrongPixels[0] += 1
def impl_tearDown(self): self.target.kill_application(self.testClient) self.target.kill_application(self.renderer) self.target.kill_application(self.ramsesDaemon) log.info("all applications killed") self.save_application_output(self.testClient) self.save_application_output(self.renderer) self.save_application_output(self.ramsesDaemon) log.info("output saved")
def is_initialised(self, timeout=DEFAULT_WAIT_FOR_MESSAGE_TIMEOUT): if self.initialisationMessage and not self.initialised and self.initialisationWatchID: log.info("waiting for initialisation message") self.initialised = self.wait_for_msg_in_stdout( self.initialisationWatchID, self.initialisationMessage, timeout) if self.initialised: log.info("initialisation message received") return self.initialised
def compare(x, y, c1, c2, diff): if all(e < percentageOfRGBDifferenceRequiredPerPixel for e in diff): if nrTooSimilarPixels[0] == 0: log.info( "First wrong Pixel {}/{} (source pixel {} / otherPixel {})" .format(x, y, c1, c2)) nrTooSimilarPixels[0] += 1 if all(e == 0 for e in diff): nrEqualPixels[0] += 1
def wait_on_surfaces_beeing_registered_in_scc(self): surfaceIdsList = list(self.expectedSurfaceIds) surfaceIdsList.sort(key=int) surfaceIdSearchRegEx = "SystemCompositorController_Wayland_IVI::listIVISurfaces Known ivi-ids are: {0}\n".format(" (([0-9])* )*".join(surfaceIdsList)) log.info("waiting on surfaces beeing registered in scc " + surfaceIdSearchRegEx) for i in xrange(1, 30): if self.renderer.send_ramsh_command("scl", response_message=surfaceIdSearchRegEx, timeout=1): return 1 return 0
def impl_tearDown(self): self.target.kill_application(self.combinedLocalClientRenderer) self.target.kill_application(self.ramsesDaemon) self.target.kill_application(self.standAloneRenderer) log.info("all applications killed") self.save_application_output(self.combinedLocalClientRenderer) self.save_application_output(self.ramsesDaemon) self.save_application_output(self.standAloneRenderer) log.info("output saved")
def impl_tearDown(self): self.targets[0].kill_application(self.HU_HMI) self.targets[0].kill_application(self.ramsesDaemon) self.targets[1].kill_application(self.KOMBI) log.info("all applications killed") self.save_application_output_on_target(self.HU_HMI, self.targets[0], nr=1) self.save_application_output_on_target(self.ramsesDaemon, self.targets[0]) self.save_application_output_on_target(self.KOMBI, self.targets[1], nr=2) log.info("output saved")
def impl_tearDown(self): self.targets[0].kill_application(self.sender) self.targets[0].kill_application(self.ramsesDaemon) self.targets[1].kill_application(self.receiver) log.info("all applications killed") self.save_application_output_on_target(self.sender, self.targets[0]) self.save_application_output_on_target(self.ramsesDaemon, self.targets[0]) self.save_application_output_on_target(self.receiver, self.targets[1]) log.info("output saved")
def impl_tearDown(self): self.target.kill_application(self.testClient) self.target.kill_application(self.rendererGreenBackground) self.target.kill_application(self.ramsesDaemon) log.info("all applications killed") self.save_application_output(self.testClient) self.save_application_output(self.rendererGreenBackground, nr=0) self.save_application_output(self.offscreenRenderer, nr=1) self.save_application_output(self.ramsesDaemon) log.info("output saved")
def impl_tearDown(self): self.target.ivi_control.cleanup() self.target.kill_application(self.testClient3Triangles) self.target.kill_application(self.rendererbackground) self.target.kill_application(self.renderer) self.target.kill_application(self.ramsesDaemon) log.info("all applications killed") self.save_application_output(self.testClient3Triangles) self.save_application_output(self.renderer) self.save_application_output(self.ramsesDaemon) log.info("output saved")
def validateScreenshot(self, renderer, imageName, displayNumber=0, useSystemCompositorForScreenshot=False, compareForEquality=True): log.info("Validating test via screenshot comparison") self.validateScreenshotOnTarget(renderer, imageName, self.target, displayNumber, useSystemCompositorForScreenshot, compareForEquality)
def _take_renderer_screenshot(self, screenshotName, renderer, displayNumber): log.info("Make screenshot of renderer") ramshCommand = "screenshot -filename \"{0}\"".format(self.tmpDir+"/"+screenshotName) if displayNumber: ramshCommand += " -displayId {}".format(displayNumber) #waitForRendererConfirmation cannot be used for screenshot creation as screenshots are enqeued at the renderer and taken on the next renderer loop result = renderer.send_ramsh_command(ramshCommand, response_message="screenshot successfully saved to file") if not result: log.warning("Screenshot confirmation not received from the renderer, check renderer application output") #print some debug output renderer.send_ramsh_command("rinfo all -v")
def impl_tearDown(self): if self.target.systemCompositorControllerSupported: self.target.ivi_control.cleanup() self.target.kill_application(self.testClient) self.target.kill_application(self.renderer) self.target.kill_application(self.ramsesDaemon) log.info("all applications killed") self.save_application_output(self.testClient) self.save_application_output(self.renderer) self.save_application_output(self.ramsesDaemon) log.info("output saved")
def createTelnetConnection(self): log.info("Connecting to power device {0} user: {1}".format( self.url, self.username)) #create telnet connection to power outlet try: telnet_conn = telnetlib.Telnet(self.url, 1234) return telnet_conn except: log.error( "Connection to power device {0} could not be established". format(self.url)) return None