class TrafficAnnotationTestsChecker(): def __init__(self, build_path=None): """Initializes a TrafficAnnotationTestsChecker object. Args: build_path: str Absolute or relative path to a fully compiled build directory. """ self.tools = NetworkTrafficAnnotationTools(build_path) def RunAllTests(self): result = self.RunOnAllFiles() #TODO(rhalavati): Add more tests, and create a pipeline for them. return result def RunOnAllFiles(self): args = ["--test-only"] _, stderr_text, return_code = self.tools.RunAuditor(args) if not return_code: print("RunOnAlFiles Passed.") elif stderr_text: print(stderr_text) return return_code
class NetworkTrafficAnnotationChecker(): EXTENSIONS = ['.cc', '.mm', '.java'] ANNOTATIONS_FILE = 'annotations.xml' def __init__(self, build_path=None): """Initializes a NetworkTrafficAnnotationChecker object. Args: build_path: str Absolute or relative path to a fully compiled build directory. If not specified, the script tries to find it based on relative position of this file (src/tools/traffic_annotation). """ self.tools = NetworkTrafficAnnotationTools(build_path) def IsAnnotationsFile(self, file_path): """Returns true if the given file is the annotations file.""" return os.path.basename(file_path) == self.ANNOTATIONS_FILE def ShouldCheckFile(self, file_path): """Returns true if the input file has an extension relevant to network traffic annotations.""" return os.path.splitext(file_path)[1] in self.EXTENSIONS def GetFilePaths(self, complete_run, limit): if complete_run: return [] # Get list of modified files. If failed, silently ignore as the test is # run in error resilient mode. file_paths = self.tools.GetModifiedFiles() or [] annotations_file_changed = any( self.IsAnnotationsFile(file_path) for file_path in file_paths) # If the annotations file has changed, trigger a full test to avoid # missing a case where the annotations file has changed, but not the # corresponding file, causing a mismatch that is not detected by just # checking the changed .cc and .mm files. if annotations_file_changed: return [] file_paths = [ file_path for file_path in file_paths if self.ShouldCheckFile(file_path) ] if not file_paths: return None # If the number of changed files in the CL exceeds a threshold, trigger # full test to avoid sending very long list of arguments and possible # failure in argument buffers. if len(file_paths) > CHANGELIST_SIZE_TO_TRIGGER_FULL_TEST: file_paths = [] return file_paths def CheckFiles(self, complete_run, limit, use_python_auditor): """Passes all given files to traffic_annotation_auditor to be checked for possible violations of network traffic annotation rules. Args: complete_run: bool Flag requesting to run test on all relevant files. use_python_auditor: bool If True, test auditor.py instead of t_a_auditor.exe. limit: int The upper threshold for number of errors and warnings. Use 0 for unlimited. Returns: int Exit code of the network traffic annotation auditor. """ if not self.tools.CanRunAuditor(use_python_auditor): print( "Network traffic annotation presubmit check was not performed. A " "compiled build directory and traffic_annotation_auditor binary " "are required to do it.") return 0 file_paths = self.GetFilePaths(complete_run, limit) if file_paths is None: return 0 args = ["--test-only", "--limit=%i" % limit, "--error-resilient"] + \ file_paths stdout_text, stderr_text, return_code = self.tools.RunAuditor( args, use_python_auditor) if stdout_text: print(stdout_text) if stderr_text: print("\n[Runtime Messages]:\n%s" % stderr_text) if self.tools.GetCurrentPlatform() == "android": # For now, always mark the android bot as green. This acts as a sort of # "FYI" mode. # # TODO(crbug.com/1254719): Once the Android presubmit bot is stable, turn # this into a CQ-blocking failure. return 0 return return_code
class TrafficAnnotationTestsChecker(): def __init__(self, build_path=None, annotations_filename=None): """Initializes a TrafficAnnotationTestsChecker object. Args: build_path: str Absolute or relative path to a fully compiled build directory. """ self.tools = NetworkTrafficAnnotationTools(build_path) self.last_result = None self.persist_annotations = bool(annotations_filename) if not annotations_filename: annotations_file = tempfile.NamedTemporaryFile() annotations_filename = annotations_file.name annotations_file.close() self.annotations_filename = annotations_filename def RunAllTests(self): """Runs all tests and returns the result.""" return self.CheckAuditorResults() and self.CheckOutputExpectations() def CheckAuditorResults(self): """Runs auditor using different configurations, expecting to run error free, and having equal results in the exported TSV file in all cases. The TSV file provides a summary of all annotations and their content. Returns: bool True if all results are as expected. """ configs = [ # Similar to trybot. [ "--test-only", "--error-resilient", ], # Failing on any runtime error. [ "--test-only", ], # No heuristic filtering. [ "--test-only", "--no-filtering", ], ] self.last_result = None for config in configs: result = self._RunTest(config, USE_PYTHON_AUDITOR) if not result: print("No output for config: %s" % config) return False if self.last_result and self.last_result != result: print("Unexpected different results for config: %s" % config) return False self.last_result = result return True def CheckOutputExpectations(self): # This test can be replaced by getting results from a diagnostic mode call # to traffic_annotation_auditor, and checking for an expected minimum number # of items for each type of pattern that it extracts. E.g., we should have # many annotations of each type (complete, partial, ...), functions that # need annotations, direct assignment to mutable annotations, etc. # |self.last_result| includes the content of the TSV file that the auditor # generates. Counting the number of end of lines in the text will give the # number of extracted annotations. annotations_count = self.last_result.count("\n") print("%i annotations found in auditor's output." % annotations_count) if annotations_count < MINIMUM_EXPECTED_NUMBER_OF_ANNOTATIONS: print("Annotations are expected to be at least %i." % MINIMUM_EXPECTED_NUMBER_OF_ANNOTATIONS) return False return True def _RunTest(self, args, use_python_auditor): """Runs the auditor test with given |args|, and returns the extracted annotations. Args: args: list of str Arguments to be passed to auditor. use_python_auditor: If True, test auditor.py instead of traffic_annotation_auditor.exe. Returns: str Content of annotations.tsv file if successful, otherwise None. """ if use_python_auditor: auditor_name = "auditor.py" else: auditor_name = "traffic_annotation_auditor" print("Running %s using config: %s" % (auditor_name, args)) try: os.remove(self.annotations_filename) except OSError: pass stdout_text, stderr_text, return_code = self.tools.RunAuditor( args + ["--annotations-file=%s" % self.annotations_filename], use_python_auditor) annotations = None if os.path.exists(self.annotations_filename): # When tests are run on all files (without filtering), there might be some # compile errors in irrelevant files on Windows that can be ignored. if (return_code and "--no-filtering" in args and sys.platform.startswith(('win', 'cygwin'))): print("Ignoring return code: %i" % return_code) return_code = 0 if not return_code: annotations = open(self.annotations_filename).read() if not self.persist_annotations: os.remove(self.annotations_filename) if annotations: print("Test PASSED.") else: print("Test FAILED.") if stdout_text: print(stdout_text) if stderr_text: print(stderr_text) return annotations
class TrafficAnnotationTestsChecker(): def __init__(self, build_path=None): """Initializes a TrafficAnnotationTestsChecker object. Args: build_path: str Absolute or relative path to a fully compiled build directory. """ self.tools = NetworkTrafficAnnotationTools(build_path) def RunAllTests(self): """Runs all tests and returns the result.""" return self.CheckAuditorResults() and self.CheckOutputExpectations() def CheckAuditorResults(self): """Runs auditor using different configurations, expecting to run error free, and having equal results in the exported TSV file in all cases. The TSV file provides a summary of all annotations and their content. Returns: bool True if all results are as expected. """ configs = [ ["--test-only", "--error-resilient"], # Similar to trybot. ["--test-only"], # Failing on any runtime error. ["--test-only", "--no-filtering"] # Not using heuristic filtering. ] last_result = None for config in configs: result = self._RunTest(config) if not result: print("No output for config: %s" % config) return False if last_result and last_result != result: print("Unexpected different results for config: %s" % config) return False last_result = result return True def CheckOutputExpectations(self): # TODO(https://crbug.com/690323): Add tests to check for an expected minimum # number of items for each type of pattern that auditor extracts. E.g., we # should have many annotations of each type (complete, partial, ...), # functions that need annotations, direct assignment to mutable annotations, # etc. return True def _RunTest(self, args): """Runs the auditor test with given |args|, and returns the extracted annotations. Args: args: list of str Arguments to be passed to auditor. Returns: str Content of annotations.tsv file if successful, otherwise None. """ print("Running auditor using config: %s" % args) temp_file = tempfile.NamedTemporaryFile() temp_filename = temp_file.name temp_file.close() _, stderr_text, return_code = self.tools.RunAuditor( args + ["--annotations-file=%s" % temp_filename]) if os.path.exists(temp_filename): annotations = None if return_code else open(temp_filename).read() os.remove(temp_filename) else: annotations = None if annotations: print("Test PASSED.") else: print("Test FAILED.\n%s" % stderr_text) return annotations
class NetworkTrafficAnnotationChecker(): EXTENSIONS = [ '.cc', '.mm', ] def __init__(self, build_path=None): """Initializes a NetworkTrafficAnnotationChecker object. Args: build_path: str Absolute or relative path to a fully compiled build directory. If not specified, the script tries to find it based on relative position of this file (src/tools/traffic_annotation). """ self.tools = NetworkTrafficAnnotationTools(build_path) def ShouldCheckFile(self, file_path): """Returns true if the input file has an extension relevant to network traffic annotations.""" return os.path.splitext(file_path)[1] in self.EXTENSIONS def CheckFiles(self, complete_run, limit): """Passes all given files to traffic_annotation_auditor to be checked for possible violations of network traffic annotation rules. Args: complete_run: bool Flag requesting to run test on all relevant files. limit: int The upper threshold for number of errors and warnings. Use 0 for unlimited. Returns: int Exit code of the network traffic annotation auditor. """ if not self.tools.CanRunAuditor(): print( "Network traffic annotation presubmit check was not performed. A " "compiled build directory and traffic_annotation_auditor binary " "are required to do it.") return 0 if complete_run: file_paths = [] else: # Get list of modified files. If failed, silently ignore as the test is # run in error resilient mode. file_paths = self.tools.GetModifiedFiles() or [] file_paths = [ file_path for file_path in file_paths if self.ShouldCheckFile(file_path) ] if not file_paths: return 0 args = ["--test-only", "--limit=%i" % limit, "--error-resilient"] + \ file_paths stdout_text, stderr_text, return_code = self.tools.RunAuditor(args) if stdout_text: print(stdout_text) if stderr_text: print("\n[Runtime Messages]:\n%s" % stderr_text) return return_code