def afterActionsRun(self, testingStep, executionSessions, traces): trace = [trace for trace in traces if trace is not None][0] self.listOfTimesForScreenshot.append(trace.timeForScreenshot) self.listOfTimesForActionMapRetrieval.append( trace.timeForActionMapRetrieval) self.listOfTimesForActionDecision.append(trace.timeForActionDecision) self.listOfTimesForActionExecution.append(trace.timeForActionExecution) self.listOfTimesForMiscellaneous.append(trace.timeForMiscellaneous) totalTime = trace.timeForScreenshot + \ trace.timeForActionMapRetrieval + \ trace.timeForActionDecision + \ trace.timeForActionExecution + \ trace.timeForMiscellaneous self.listOfTotalLoopTimes.append(totalTime) if trace.traceNumber % self.config['testing_print_every'] == ( self.config['testing_print_every'] - 1) or trace.traceNumber == 0: msg = f"Finished {trace.traceNumber + 1} testing actions." if len(self.listOfTimesForScreenshot): msg += f"\n Avg Screenshot time: {numpy.average(self.listOfTimesForScreenshot[-self.config['testing_print_every']:]):.4f}" msg += f"\n Avg Action Map Retrieval Time: {numpy.average(self.listOfTimesForActionMapRetrieval[-self.config['testing_print_every']:]):.4f}" msg += f"\n Avg Action Decision Time: {numpy.average(self.listOfTimesForActionDecision[-self.config['testing_print_every']:]):.4f}" msg += f"\n Avg Action Execution Time: {numpy.average(self.listOfTimesForActionExecution[-self.config['testing_print_every']:]):.4f}" msg += f"\n Avg Miscellaneous Time: {numpy.average(self.listOfTimesForMiscellaneous[-self.config['testing_print_every']:]):.4f}" msg += f"\n Avg Total Loop Time: {numpy.average(self.listOfTotalLoopTimes[-self.config['testing_print_every']:]):.4f}" getLogger().info(msg)
def testingStepFinished(self, testingStep, executionSessions): if self.config[ 'precompute_sample_cache_single_threaded'] or self.config[ 'precompute_sample_cache_num_workers'] == 1: for session in executionSessions: getLogger().info( f"Preparing samples for {session.id} and adding them to the sample cache." ) TrainingManager.addExecutionSessionToSampleCache( session.id, self.config) else: # For some reason, we are getting frequent errors with the multi-process # based version of this code shown below in our cloud environment. with concurrent.futures.ProcessPoolExecutor( max_workers=self. config['precompute_sample_cache_num_workers']) as executor: futures = [] for session in executionSessions: getLogger().info( f"Preparing samples for {session.id} and adding them to the sample cache." ) futures.append( executor.submit( PrecomputeSessionsForSampleCache. addExecutionSessionToSampleCache, session.id, self.config)) for future in futures: future.result()
def getResourceLink(resourceURL, baseURL, resourceDataModifyFunc=None): resourceURL = urlWithoutFragment( urllib.parse.urljoin(baseURL, resourceURL)) versionId, data = proxy.getResourceVersion(resourceURL) if versionId is None: try: proxies = { 'http': f'http://127.0.0.1:{proxy.port}', 'https': f'http://127.0.0.1:{proxy.port}', } response = requests.get(resourceURL, proxies=proxies, verify=False) versionId, data = proxy.getResourceVersion(resourceURL) except requests.exceptions.RequestException: pass if data is None: if resourceURL not in self.failedResourceUrls: getLogger().warning( f"WARNING! Failed to grab the resource at URL: {resourceURL}" ) self.failedResourceUrls.add(resourceURL) return None else: # urlPath = urllib.parse.urlparse(resourceURL).path # extension = os.path.splitext(urlPath)[1] # if resourceDataModifyFunc is not None: # data = resourceDataModifyFunc(data, resourceURL) return f"kwolaResourceVersion://{versionId}"
def encodeAndSaveVideo(self, executionSession, folder, lossless): codec = chooseBestFfmpegVideoCodec(losslessPreferred=lossless) crfRating = 25 if lossless: crfRating = 0 result = subprocess.run( [ 'ffmpeg', '-f', 'image2', "-r", "1", '-i', 'kwola-screenshot-%05d.png', '-vcodec', codec, '-crf', str(crfRating), '-preset', 'veryslow', self.movieFileName(executionSession) ], cwd=self.screenshotDirectory[executionSession.id], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: errorMsg = f"Error! Attempted to create a movie using ffmpeg and the process exited with exit-code {result.returncode}. The following output was observed:\n" errorMsg += str(result.stdout, 'utf8') + "\n" errorMsg += str(result.stderr, 'utf8') + "\n" getLogger().error(errorMsg) raise RuntimeError(errorMsg) else: localVideoPath = self.movieFilePath(executionSession) with open(localVideoPath, 'rb') as f: videoData = f.read() fileName = f'{str(executionSession.id)}.mp4' self.config.saveKwolaFileData(folder, fileName, videoData) os.unlink(localVideoPath)
def testingStepFinished(self, testingStep, executionSessions): pool = multiprocessing.Pool( self.config['testing_video_generation_processes'], maxtasksperchild=1) futures = [] for session in executionSessions: future = pool.apply_async(func=createDebugVideoSubProcess, args=(self.config.serialize(), str(session.id), "", False, False, None, None, "annotated_videos")) futures.append((session, future)) for session, future in futures: localFuture = future # for retry in range(5): # try: value = localFuture.get( timeout=self.config['debug_video_generation_timeout']) if value: getLogger().error(value) # break # except billiard.exceptions.WorkerLostError: # if retry == 4: # raise # localFuture = pool.apply_async(func=createDebugVideoSubProcess, # args=(self.config.serialize(), str(session.id), "", False, False, None, None, "annotated_videos")) # except BrokenPipeError: # if retry == 4: # raise # localFuture = pool.apply_async(func=createDebugVideoSubProcess, # args=(self.config.serialize(), str(session.id), "", False, False, None, None, "annotated_videos")) pool.close() pool.join()
def extractBranchTrace(self, webDriver): # The JavaScript that we want to inject. This will extract out the Kwola debug information. if isinstance(webDriver, Firefox): injected_javascript = ( 'const newCounters = {};' 'if (window.kwolaCounters)' '{' ' for (const [key, value] of Object.entries(window.kwolaCounters))' ' {' ' newCounters[key] = Array.from(value);' ' }' ' return newCounters;' '}' 'else' '{' ' return null;' '}') elif isinstance(webDriver, Chrome) or isinstance(webDriver, Edge): injected_javascript = ('return window.kwolaCounters;') else: raise RuntimeError("Unrecognized web driver class.") result = webDriver.execute_script(injected_javascript) # The JavaScript that we want to inject. This will extract out the Kwola debug information. injected_javascript = ( 'if (!window.kwolaCounters)' '{' ' window.kwolaCounters = {};' '}' 'Object.keys(window.kwolaCounters).forEach((fileName) => {' ' window.kwolaCounters[fileName].fill(0);' '});') try: webDriver.execute_script(injected_javascript) except selenium.common.exceptions.TimeoutException: getLogger().warning( f"Warning, timeout while running the script to reset the kwola line counters." ) if result is not None: # Cast everything to a numpy array so we don't have to do it later for fileName, vector in result.items(): result[fileName] = numpy.array(vector) else: getLogger().warning( f"Warning, did not find the kwola line counter object in the browser. This usually " "indicates that there was an error either in translating the javascript, an error " "in loading the page, or that the page has absolutely no javascript. " f"On page: {webDriver.current_url}") result = {} return result
def afterActionRuns(self, webDriver, proxy, executionSession, executionTrace, actionExecuted): exceptions = self.extractExceptions(webDriver) for exception in exceptions: msg, source, lineno, colno, stack = tuple(exception) msg = str(msg) source = str(source) stack = str(stack) combinedMessage = msg + source + stack kwolaJSRewriteErrorFound = False for detectionString in kwolaJSRewriteErrorDetectionStrings: if detectionString in combinedMessage: kwolaJSRewriteErrorFound = True if kwolaJSRewriteErrorFound: logMsgString = f"Error. There was a bug generated by the underlying javascript application, " \ f"but it appears to be a bug in Kwola's JS rewriting. Please notify the Kwola " \ f"developers that this url: {webDriver.current_url} gave you a js-code-rewriting " \ f"issue. \n" logMsgString += f"{msg} at line {lineno} column {colno} in {source}\n" logMsgString += f"{str(stack)}\n" getLogger().error(logMsgString) else: error = ExceptionError(type="exception", page=executionTrace.startURL, stacktrace=stack, message=msg, source=source, lineNumber=lineno, columnNumber=colno) executionTrace.errorsDetected.append(error) errorHash = error.computeHash() executionTrace.didErrorOccur = True if errorHash not in self.errorHashes[executionSession.id]: if errorHash not in self.allErrorHashes and not self.isDuplicate( error): logMsgString = f"An unhandled exception was detected in client application:\n" logMsgString += f"{msg} at line {lineno} column {colno} in {source}\n" logMsgString += f"{str(stack)}" getLogger().info(logMsgString) self.allErrorHashes.add(errorHash) self.allErrors.append(error) self.errorHashes[executionSession.id].add(errorHash) executionTrace.didNewErrorOccur = True
def afterActionRuns(self, webDriver, proxy, executionSession, executionTrace, actionExecuted): branchTrace = self.extractBranchTrace(webDriver) newBranches = False filteredBranchTrace = {} for fileName in branchTrace.keys(): traceVector = branchTrace[fileName] didExecuteFile = bool(numpy.sum(traceVector) > 0) if didExecuteFile: filteredBranchTrace[fileName] = traceVector if fileName in self.cumulativeBranchTrace[executionSession.id]: cumulativeTraceVector = self.cumulativeBranchTrace[executionSession.id][fileName] if len(traceVector) == len(cumulativeTraceVector): newBranchCount = numpy.sum(traceVector[cumulativeTraceVector == 0]) if newBranchCount > 0: newBranches = True else: if didExecuteFile: newBranches = True else: if didExecuteFile: newBranches = True executionTrace.branchTrace = {k: v.tolist() for k, v in filteredBranchTrace.items()} executionTrace.didCodeExecute = bool(len(filteredBranchTrace) > 0) executionTrace.didNewBranchesExecute = bool(newBranches) total = 0 executedAtleastOnce = 0 for fileName in self.cumulativeBranchTrace[executionSession.id]: total += len(self.cumulativeBranchTrace[executionSession.id][fileName]) executedAtleastOnce += numpy.count_nonzero(self.cumulativeBranchTrace[executionSession.id][fileName]) # Just an extra check here to cover our ass in case of division by zero if total == 0: total += 1 executionTrace.cumulativeBranchCoverage = float(executedAtleastOnce) / float(total) for fileName in filteredBranchTrace.keys(): if fileName in self.cumulativeBranchTrace[executionSession.id]: if len(branchTrace[fileName]) == len(self.cumulativeBranchTrace[executionSession.id][fileName]): self.cumulativeBranchTrace[executionSession.id][fileName] += branchTrace[fileName] else: getLogger().warning( f"Warning! The file with fileName {fileName} has changed the size of its trace vector. This " f"is very unusual and could indicate some strange situation with dynamically loaded javascript") else: self.cumulativeBranchTrace[executionSession.id][fileName] = branchTrace[fileName]
def browserSessionFinished(self, webDriver, proxy, executionSession): result = subprocess.run(['ffmpeg', '-f', 'image2', "-r", "3", '-i', 'kwola-screenshot-%05d.png', '-vcodec', chooseBestFfmpegVideoCodec(), '-pix_fmt', 'yuv420p', '-crf', '15', '-preset', 'veryslow', self.movieFileName(executionSession)], cwd=self.screenshotDirectory[executionSession.id], stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: errorMsg = f"Error! Attempted to create a movie using ffmpeg and the process exited with exit-code {result.returncode}. The following output was observed:\n" errorMsg += str(result.stdout, 'utf8') + "\n" errorMsg += str(result.stderr, 'utf8') + "\n" getLogger().error(errorMsg)
def testingStepFinished(self, testingStep, executionSessions): totalRewards = [] for session in executionSessions: getLogger().info( f"Session {session.tabNumber} finished with total reward: {session.totalReward:.3f}" ) totalRewards.append(session.totalReward) if len(totalRewards) > 0: getLogger().info( f"Mean total reward of all sessions: {numpy.mean(totalRewards):.3f}" )
def testingStepFinished(self, testingStep, executionSessions): with concurrent.futures.ProcessPoolExecutor(max_workers=2) as executor: futures = [] for session in executionSessions: getLogger().info( f"[{os.getpid()}] Preparing samples for {session.id} and adding them to the sample cache." ) futures.append( executor.submit( TrainingManager.addExecutionSessionToSampleCache, session.id, self.config)) for future in futures: future.result()
def willRewriteFile(self, url, contentType, fileData): jsMimeTypes = [ "application/x-javascript", "application/javascript", "application/ecmascript", "text/javascript", "text/ecmascript" ] cleanedFileName = self.getCleanedFileName(url) if ('_js' in url and not "_json" in url and not "_jsp" in url and not url.endswith("_css") ) or str(contentType).strip().lower() in jsMimeTypes: kind = filetype.guess(fileData) mime = '' if kind is not None: mime = kind.mime # Next, check to see that we haven't gotten an image or something else that we should ignore. This happens, surprisingly. if mime.startswith("image/") or mime.startswith( "video/") or mime.startswith("audio/") or mime.startswith( "application/"): return False # For some reason, some websites send JSON data in files labelled as javascript files. # So we have to double check to make sure we aren't looking at JSON data try: json.loads(str(fileData, 'utf8').lower()) return False except json.JSONDecodeError: pass except UnicodeDecodeError: pass if fileData.startswith(b"<html>"): return False ignoreKeyword = self.findMatchingJavascriptFilenameIgnoreKeyword( cleanedFileName) if ignoreKeyword is None: return True else: getLogger().info( f"[{os.getpid()}] Warning: Ignoring the javascript file '{cleanedFileName}' because it matches the javascript ignore keyword '{ignoreKeyword}'. " f"This means that no learnings will take place on the code in this file. If this file is actually part of your " f"application and should be learned on, then please modify your config file kwola.json and remove the ignore " f"keyword '{ignoreKeyword}' from the variable 'web_session_ignored_javascript_file_keywords'. This file will be " f"cached without Kwola line counting installed. Its faster to install line counting only in the files that need " f"it.") return False else: return False
def afterActionRuns(self, webDriver, proxy, executionSession, executionTrace, actionExecuted): logEntries = webDriver.get_log( 'browser')[self.startLogCounts[executionSession.id]:] for log in logEntries: if log['level'] == 'SEVERE': message = str(log['message']) message = message.replace("\\n", "\n") # If it looks like a network error, then ignore it because those are handled separately if RecordLogEntriesAndLogErrors.networkErrorRegex.search( message) is not None: continue kwolaJSRewriteErrorFound = False for detectionString in kwolaJSRewriteErrorDetectionStrings: if detectionString in message: kwolaJSRewriteErrorFound = True if kwolaJSRewriteErrorFound: logMsgString = f"[{os.getpid()}] Error. There was a bug generated by the underlying javascript application, " \ f"but it appears to be a bug in Kwola's JS rewriting. Please notify the Kwola " \ f"developers that this url: {webDriver.current_url} gave you a js-code-rewriting " \ f"issue.\n" logMsgString += f"{message}\n" getLogger().error(logMsgString) else: error = LogError(type="log", page=executionTrace.startURL, message=message, logLevel=log['level']) executionTrace.errorsDetected.append(error) errorHash = error.computeHash() executionTrace.didErrorOccur = True if errorHash not in self.errorHashes[executionSession.id]: logMsgString = f"[{os.getpid()}] A log error was detected in client application:\n" logMsgString += f"{message}\n" getLogger().info(logMsgString) self.errorHashes[executionSession.id].add(errorHash) executionTrace.didNewErrorOccur = True executionTrace.logOutput = "\n".join([str(log) for log in logEntries]) executionTrace.hadLogOutput = bool(executionTrace.logOutput)
def afterActionRuns(self, webDriver, proxy, executionSession, executionTrace, actionExecuted): for networkError in proxy.getNetworkErrors(): networkError.page = executionTrace.startURL executionTrace.errorsDetected.append(networkError) errorHash = networkError.computeHash() if errorHash not in self.errorHashes[executionSession.id]: networkErrorMsgString = f"[{os.getpid()}] A network error was detected in client application:\n" networkErrorMsgString += f"Path: {networkError.path}\n" networkErrorMsgString += f"Status Code: {networkError.statusCode}\n" networkErrorMsgString += f"Message: {networkError.message}\n" getLogger().info(networkErrorMsgString) self.errorHashes[executionSession.id].add(errorHash) executionTrace.didNewErrorOccur = True
def generateVideoFilesForBugs(self, testingStep, bugObjects): pool = multiprocessing.Pool( self.config['testing_video_generation_processes'], maxtasksperchild=1) futures = [] for bugIndex, bug in enumerate(bugObjects): future = pool.apply_async(func=createDebugVideoSubProcess, args=(self.config.serialize(), str(bug.executionSessionId), f"{bug.id}_bug", False, False, bug.stepNumber, bug.stepNumber + 3, "bugs")) futures.append((bugIndex, bug, future)) for bugIndex, bug, future in futures: localFuture = future # for retry in range(5): # try: value = localFuture.get( timeout=self.config['debug_video_generation_timeout']) if value: getLogger().error(value) # break # except billiard.exceptions.WorkerLostError: # if retry == 4: # raise # localFuture = pool.apply_async(func=createDebugVideoSubProcess, args=( # self.config.serialize(), str(bug.executionSessionId), f"{bug.id}_bug", False, False, bug.stepNumber, # bug.stepNumber + 3, "bugs")) # except BrokenPipeError: # if retry == 4: # raise # localFuture = pool.apply_async(func=createDebugVideoSubProcess, args=( # self.config.serialize(), str(bug.executionSessionId), f"{bug.id}_bug", False, False, bug.stepNumber, # bug.stepNumber + 3, "bugs")) pool.close() pool.join()
def extractBranchTrace(self, webDriver): # The JavaScript that we want to inject. This will extract out the Kwola debug information. injected_javascript = ( 'return window.kwolaCounters;' ) result = webDriver.execute_script(injected_javascript) # The JavaScript that we want to inject. This will extract out the Kwola debug information. injected_javascript = ( 'if (!window.kwolaCounters)' '{' ' window.kwolaCounters = {};' '}' 'Object.keys(window.kwolaCounters).forEach((fileName) => {' ' window.kwolaCounters[fileName].fill(0);' '});' ) try: webDriver.execute_script(injected_javascript) except selenium.common.exceptions.TimeoutException: getLogger().warning(f"[{os.getpid()}] Warning, timeout while running the script to reset the kwola line counters.") if result is not None: # Cast everything to a numpy array so we don't have to do it later for fileName, vector in result.items(): result[fileName] = numpy.array(vector) else: getLogger().warning(f"[{os.getpid()}] Warning, did not find the kwola line counter object in the browser. This usually " "indicates that there was an error either in translating the javascript, an error " "in loading the page, or that the page has absolutely no javascript. " f"On page: {webDriver.current_url}") result = {} return result
def rewriteFile(self, url, contentType, fileData): jsFileContents = fileData.strip() strictMode = False if jsFileContents.startswith( b"'use strict';") or jsFileContents.startswith( b'"use strict";'): strictMode = True jsFileContents = jsFileContents.replace(b"'use strict';", b"") jsFileContents = jsFileContents.replace(b'"use strict";', b"") wrapperStart = b"" wrapperEnd = b"" for wrapper in JSRewriter.knownResponseWrappers: if jsFileContents.startswith( wrapper[0]) and jsFileContents.endswith(wrapper[1]): jsFileContents = jsFileContents[len(wrapper[0] ):-len(wrapper[1])] wrapperStart = wrapper[0] wrapperEnd = wrapper[1] cleanedFileName = self.getCleanedFileName(url) longFileHash, shortFileHash = ProxyPluginBase.computeHashes( bytes(fileData)) fileNameForBabel = shortFileHash + "_" + cleanedFileName environment = dict(os.environ) environment['KWOLA_ENABLE_LINE_COUNTING'] = 'true' environment['KWOLA_ENABLE_EVENT_HANDLER_TRACKING'] = 'true' noLineCountingKeyword = self.findMatchingJavascriptFilenameNoLineCountingKeyword( cleanedFileName) if noLineCountingKeyword is not None: environment['KWOLA_ENABLE_LINE_COUNTING'] = 'false' getLogger().info( f"Warning: Not installing line counting in the javascript file '{cleanedFileName}' because it matches the " f"javascript no line counting keyword '{noLineCountingKeyword}'. Event handler tracking will still be installed." f"This means that no learnings will take place on the code in this file. If this file is actually part of your " f"application and should be learned on, then please modify your config file kwola.json and remove the ignore " f"keyword '{noLineCountingKeyword}' from the variable 'web_session_no_line_counting_javascript_file_keywords'. This file will be " f"cached without Kwola line counting installed. Its faster to install line counting only in the files that need " f"it.") babelCmd = 'babel' if sys.platform == "win32" or sys.platform == "win64": babelCmd = 'babel.cmd' result = subprocess.run([ babelCmd, '-f', fileNameForBabel, '--plugins', 'babel-plugin-kwola', '--retain-lines', '--source-type', "script" ], input=jsFileContents, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment) if result.returncode != 0 and "'import' and 'export' may appear only with" in str( result.stderr, 'utf8'): result = subprocess.run([ babelCmd, '-f', fileNameForBabel, '--plugins', 'babel-plugin-kwola', '--retain-lines', '--source-type', "module" ], input=jsFileContents, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environment) if result.returncode != 0: cutoffLength = 250 kind = filetype.guess(fileData) mime = '' if kind is not None: mime = kind.mime getLogger().warning( f"Unable to install Kwola line-counting in the Javascript file {url}. Most" f" likely this is because Babel thinks your javascript has invalid syntax, or that" f" babel is not working / not able to find the babel-plugin-kwola / unable to" f" transpile the javascript for some other reason. See the following truncated" f" output:") if len(result.stdout) > 0: getLogger().warning(result.stdout[:cutoffLength]) else: getLogger().warning("No data in standard output") if len(result.stderr) > 0: getLogger().warning(result.stderr[:cutoffLength]) else: getLogger().warning("No data in standard error output") return fileData else: # Check to see if the resulting file object had multiple branches if noLineCountingKeyword is None and not self.checkIfRewrittenJSFileHasMultipleBranches( result.stdout): getLogger().warning( f"Ignoring the javascript file {url} because it looks like a JSONP-style request, or some other javascript " f"file without a significant number of code branches.") return fileData getLogger().info( f"Successfully translated {url} with Kwola modifications.") transformed = wrapperStart + result.stdout + wrapperEnd if strictMode: transformed = b'"use strict";\n' + transformed return transformed
def browserSessionFinished(self, webDriver, proxy, executionSession): if executionSession.bestApplicationProvidedCumulativeFitness is not None: getLogger().info(f"Session {executionSession.id} finished with fitness: {executionSession.bestApplicationProvidedCumulativeFitness:.0f}")
def rewriteFile(self, url, contentType, fileData): jsFileContents = fileData.strip() strictMode = False if jsFileContents.startswith( b"'use strict';") or jsFileContents.startswith( b'"use strict";'): strictMode = True jsFileContents = jsFileContents.replace(b"'use strict';", b"") jsFileContents = jsFileContents.replace(b'"use strict";', b"") wrapperStart = b"" wrapperEnd = b"" for wrapper in JSRewriter.knownResponseWrappers: if jsFileContents.startswith( wrapper[0]) and jsFileContents.endswith(wrapper[1]): jsFileContents = jsFileContents[len(wrapper[0] ):-len(wrapper[1])] wrapperStart = wrapper[0] wrapperEnd = wrapper[1] cleanedFileName = self.getCleanedFileName(url) longFileHash, shortFileHash = ProxyPluginBase.computeHashes( bytes(fileData)) fileNameForBabel = shortFileHash + "_" + cleanedFileName result = subprocess.run([ 'babel', '-f', fileNameForBabel, '--plugins', 'babel-plugin-kwola', '--retain-lines', '--source-type', "script" ], input=jsFileContents, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0 and "'import' and 'export' may appear only with" in str( result.stderr, 'utf8'): result = subprocess.run([ 'babel', '-f', fileNameForBabel, '--plugins', 'babel-plugin-kwola', '--retain-lines', '--source-type', "module" ], input=jsFileContents, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if result.returncode != 0: cutoffLength = 250 kind = filetype.guess(fileData) mime = '' if kind is not None: mime = kind.mime getLogger().warning( f"[{os.getpid()}] Unable to install Kwola line-counting in the Javascript file {url}. Most" f" likely this is because Babel thinks your javascript has invalid syntax, or that" f" babel is not working / not able to find the babel-plugin-kwola / unable to" f" transpile the javascript for some other reason. See the following truncated" f" output:") if len(result.stdout) > 0: getLogger().warning(result.stdout[:cutoffLength]) else: getLogger().warning("No data in standard output") if len(result.stderr) > 0: getLogger().warning(result.stderr[:cutoffLength]) else: getLogger().warning("No data in standard error output") return fileData else: getLogger().info( f"[{os.getpid()}] Successfully translated {url} with Kwola branch counting and event tracing." ) transformed = wrapperStart + result.stdout + wrapperEnd if strictMode: transformed = b'"use strict";\n' + transformed return transformed
def afterActionRuns(self, webDriver, proxy, executionSession, executionTrace, actionExecuted): if isinstance(webDriver, Firefox): logEntries = [] rawLogs = webDriver.execute_script( """const logs = window.kwolaLogs; window.kwolaLogs = []; return logs;""" ) if rawLogs is not None: for entry in rawLogs: logEntries.append({ "level": entry[0], "message": " ".join([str(m) for m in entry[1]]) }) elif isinstance(webDriver, Chrome) or isinstance(webDriver, Edge): logEntries = webDriver.get_log( 'browser')[self.startLogCounts[executionSession.id]:] else: raise RuntimeError("Unrecognized web driver class") for log in logEntries: if log['level'] == 'SEVERE' or log['level'] == 'error': message = str(log['message']) message = message.replace("\\n", "\n") # If it looks like a network error, then ignore it because those are handled separately if RecordLogEntriesAndLogErrors.networkErrorRegex.search( message) is not None: continue kwolaJSRewriteErrorFound = False for detectionString in kwolaJSRewriteErrorDetectionStrings: if detectionString in message: kwolaJSRewriteErrorFound = True break ignoreErrorKeywordFound = False matchingIgnoreKeyword = None for ignoreKeyword in self.config[ 'web_session_ignored_log_error_keywords']: if ignoreKeyword in message: ignoreErrorKeywordFound = True matchingIgnoreKeyword = ignoreKeyword break if kwolaJSRewriteErrorFound: logMsgString = f"Error. There was a bug generated by the underlying javascript application, " \ f"but it appears to be a bug in Kwola's JS rewriting. Please notify the Kwola " \ f"developers that this url: {webDriver.current_url} gave you a js-code-rewriting " \ f"issue.\n" logMsgString += f"{message}\n" getLogger().error(logMsgString) elif ignoreErrorKeywordFound: logMsgString = f"Suppressed an error message because it matched the " \ f"error ignore keyword {matchingIgnoreKeyword}. " logMsgString += f"{message}\n" getLogger().info(logMsgString) else: error = LogError(type="log", page=executionTrace.startURL, message=message, logLevel=log['level']) executionTrace.errorsDetected.append(error) errorHash = error.computeHash() executionTrace.didErrorOccur = True if errorHash not in self.errorHashes[executionSession.id]: if errorHash not in self.allErrorHashes and not self.isDuplicate( error): logMsgString = f"A log error was detected in client application:\n" logMsgString += f"{message}\n" getLogger().info(logMsgString) self.allErrorHashes.add(errorHash) self.allErrors.append(error) self.errorHashes[executionSession.id].add(errorHash) executionTrace.didNewErrorOccur = True executionTrace.logOutput = "\n".join([str(log) for log in logEntries]) executionTrace.hadLogOutput = bool(executionTrace.logOutput)
def browserSessionFinished(self, webDriver, proxy, executionSession): getLogger().info( f"Creating movie file for the execution session {executionSession.id}" ) self.encodeAndSaveVideo(executionSession, "videos", False) self.encodeAndSaveVideo(executionSession, "videos_lossless", True)
def testingStepFinished(self, testingStep, executionSessions): kwolaVideoDirectory = self.config.getKwolaUserDataDirectory("videos") existingBugs = self.loadAllBugs(testingStep) bugObjects = [] executionSessionsById = {} for session in executionSessions: executionSessionsById[session.id] = session for errorIndex, error, executionSessionId, stepNumber in zip( range(len(self.newErrorsThisTestingStep[testingStep.id])), self.newErrorsThisTestingStep[testingStep.id], self.newErrorOriginalExecutionSessionIds[testingStep.id], self.newErrorOriginalStepNumbers[testingStep.id]): if error.type == "http": if error.statusCode == 400 and not self.config[ 'enable_400_error']: continue # Skip this error if error.statusCode == 401 and not self.config[ 'enable_401_error']: continue # Skip this error if error.statusCode == 403 and not self.config[ 'enable_403_error']: continue # Skip this error if error.statusCode == 404 and not self.config[ 'enable_404_error']: continue # Skip this error if error.statusCode >= 500 and not self.config[ 'enable_5xx_error']: continue # Skip this error elif error.type == "log": if not self.config['enable_javascript_console_error']: continue # Skip this error elif error.type == "exception": if not self.config['enable_unhandled_exception_error']: continue # Skip this error bug = BugModel() bug.owner = testingStep.owner bug.applicationId = testingStep.applicationId bug.testingStepId = testingStep.id bug.executionSessionId = executionSessionId bug.creationDate = datetime.now() bug.stepNumber = stepNumber bug.error = error bug.testingRunId = testingStep.testingRunId bug.actionsPerformed = [ trace.actionPerformed for trace in self.executionSessionTraces[executionSessionId] ][:(bug.stepNumber + 2)] bug.browser = executionSessionsById[executionSessionId].browser bug.userAgent = executionSessionsById[executionSessionId].userAgent bug.windowSize = executionSessionsById[ executionSessionId].windowSize tracesForScore = [ trace for trace in self.executionSessionTraces[executionSessionId] [max(0, stepNumber - 5):(stepNumber + 1)] if trace.codePrevalenceScore is not None ] if len(tracesForScore) > 0: bug.codePrevalenceScore = numpy.mean( [trace.codePrevalenceScore for trace in tracesForScore]) else: bug.codePrevalenceScore = None bug.isBugNew = True bug.recomputeBugQualitativeFeatures() duplicate = False for existingBug in existingBugs: if bug.isDuplicateOf(existingBug): duplicate = True break if not duplicate: bug.id = CustomIDField.generateNewUUID(BugModel, self.config) bug.saveToDisk(self.config, overrideSaveFormat="json", overrideCompression=0) bug.saveToDisk(self.config) bugTextFile = os.path.join( self.config.getKwolaUserDataDirectory("bugs"), bug.id + ".txt") saveKwolaFileData(bugTextFile, bytes(bug.generateBugText(), "utf8"), self.config) bugVideoFilePath = os.path.join( self.config.getKwolaUserDataDirectory("bugs"), bug.id + ".mp4") origVideoFilePath = os.path.join( kwolaVideoDirectory, f'{str(executionSessionId)}.mp4') origVideoFileData = loadKwolaFileData(origVideoFilePath, self.config) saveKwolaFileData(bugVideoFilePath, origVideoFileData, self.config) getLogger().info( f"\n\nBug #{errorIndex + 1}:\n{bug.generateBugText()}\n") existingBugs.append(bug) bugObjects.append(bug) getLogger().info( f"\n\nBug #{errorIndex + 1}:\n{bug.generateBugText()}\n") getLogger().info( f"Found {len(self.newErrorsThisTestingStep[testingStep.id])} new unique errors this session." ) testingStep.bugsFound = len( self.newErrorsThisTestingStep[testingStep.id]) testingStep.errors = self.newErrorsThisTestingStep[testingStep.id] self.generateVideoFilesForBugs(testingStep, bugObjects)