def _MakeShellScrubbers(self): shell_scrubbers = [] shell_scrubbers.append( comment_scrubber.CommentScrubber( comment_scrubber.ShellLikeCommentExtractor(), comment_scrubbers=self._CommentScrubbers())) shell_scrubbers.extend(self._PolyglotFileScrubbers()) return shell_scrubbers
def assertContentsError(self, lines): context = FakeContext() scrubber = comment_scrubber.CommentScrubber( comment_scrubber.CLikeCommentExtractor()) file_obj = test_util.FakeFile(contents=u'\n'.join(lines)) scrubber.ScrubFile(file_obj, context) self.assertEqual(1, len(context.errors)) error, = context.errors self.assertEqual('COMMENT_SCRUBBER_ERROR', error.filter)
def _MakeHtmlScrubbers(self): html_scrubbers = [] html_scrubbers.append( comment_scrubber.CommentScrubber( comment_scrubber.HtmlCommentExtractor(), self._CommentScrubbers())) line_scrubbers = self._PolyglotLineOrientedScrubbers() for js_directory_rename in self.js_directory_renames: line_scrubbers.append(js_directory_rename) html_scrubbers.append(line_scrubber.LineScrubber(line_scrubbers)) html_scrubbers.extend(self._PolyglotFileScrubbers()) return html_scrubbers
def RunScenario(self, scenario_name, extractor=None): if not extractor: extractor = comment_scrubber.CLikeCommentExtractor() scrubber = comment_scrubber.CommentScrubber(extractor) scenario_dir = os.path.join(TEST_DATA_DIR, 'directives', scenario_name) unstripped = os.path.join(scenario_dir, 'input.txt') file_obj = test_util.FakeFile(filename=unstripped) scrubber.ScrubFile(file_obj, None) expected = os.path.join(scenario_dir, 'expected.txt') out = os.path.join(FLAGS.test_tmpdir, scenario_name + '.out.txt') file_util.Write(out, file_obj.new_contents) basetest.DiffTestFiles(out, expected)
def __init__(self, action): """Initialize. Args: action: base.ACTION_* constant, what to do to an unmeaningful file. """ base.FileScrubber.__init__(self) # Use a separate extractor and scrubber. We don't want the scrubber to # actually scrub, but we need to extract comments to pass to # _comment_scrubber.DetermineNewContents. self._extractor = comment_scrubber.CLikeCommentExtractor() self._comment_scrubber = comment_scrubber.CommentScrubber( extractor=None, comment_scrubbers=[RemoveCommentsScrubber()]) self._action = action
def _ResetBatchScrubbers(self): """Set batch scrubbers to run before and after by-file scrubbing. See ScrubberContext.Scan below. First, the pre-batch scrubbers are run for applicable files, then by-file scrubbers, then post-batch scrubbers. ("Pre-batch" and "post-batch" are misnomers, but used for simplicity. Respectively, they're really "pre-(by-file)" and "post-(by-file)".) """ c_like_comment_pre_batch_scrubbers = [ comment_scrubber.CommentScrubber( comment_scrubber.CLikeCommentExtractor(), self._CommentScrubbers()) ] java_pre_batch_scrubbers = [] java_pre_batch_scrubbers.extend(c_like_comment_pre_batch_scrubbers) proto_pre_batch_scrubbers = (self.scrub_proto_comments and c_like_comment_pre_batch_scrubbers or []) self.extension_to_pre_batch_scrubbers_map = { '.c': c_like_comment_pre_batch_scrubbers, '.cc': c_like_comment_pre_batch_scrubbers, '.go': c_like_comment_pre_batch_scrubbers, '.h': c_like_comment_pre_batch_scrubbers, '.java': java_pre_batch_scrubbers, '.jj': java_pre_batch_scrubbers, '.js': c_like_comment_pre_batch_scrubbers, '.jslib': c_like_comment_pre_batch_scrubbers, '.l': c_like_comment_pre_batch_scrubbers, '.php': c_like_comment_pre_batch_scrubbers, '.php4': c_like_comment_pre_batch_scrubbers, '.php5': c_like_comment_pre_batch_scrubbers, '.proto': proto_pre_batch_scrubbers, '.protodevel': proto_pre_batch_scrubbers, '.swig': c_like_comment_pre_batch_scrubbers, } java_post_batch_scrubbers = [] if self.empty_java_file_action != base.ACTION_IGNORE: java_post_batch_scrubbers.append( java_scrubber.EmptyJavaFileScrubber( self.empty_java_file_action)) self.extension_to_post_batch_scrubbers_map = { '.java': java_post_batch_scrubbers, '.jj': java_post_batch_scrubbers, }
def _MakePythonScrubbers(self): py_scrubbers = [] py_scrubbers.append( comment_scrubber.CommentScrubber( comment_scrubber.PythonCommentExtractor(), self._CommentScrubbers())) line_scrubbers = [] line_scrubbers.extend(self.python_module_renames) line_scrubbers.extend(self.python_module_removes) if self.scrub_authors: line_scrubbers.append( line_scrubber.PythonAuthorDeclarationScrubber( self.username_filter)) if self.python_shebang_replace: py_scrubbers.append(self.python_shebang_replace) line_scrubbers += self._PolyglotLineOrientedScrubbers() py_scrubbers.append(line_scrubber.LineScrubber(line_scrubbers)) py_scrubbers.extend(self._PolyglotFileScrubbers()) return py_scrubbers
def testStripLineDirectiveC(self): scrubber = comment_scrubber.CommentScrubber( comment_scrubber.CLikeCommentExtractor()) self.assertContents(scrubber, ['int a;', 'int c;'], ['int a;', '// MOE:strip_line', 'int c;']) self.assertContents(scrubber, ['int a;', 'int c;'], [ 'int a;', 'int b; // MOE:strip_line The preceding relies on the magic seed 42', 'int c;' ]) self.assertContents(scrubber, ['int a;', 'int c;'], ['int a;', 'int b; // MOE:strip_line', 'int c;']) self.assertContents(scrubber, ['int a;', 'int c;'], [ 'int a;', 'int b1; // MOE:strip_line', 'int b2; // MOE:strip_line', 'int c;' ]) self.assertContents(scrubber, ['int a;', 'int c;'], [ 'int a;', 'int b; // This line and comment are secret. MOE:strip_line', 'int c;' ]) self.assertContents( scrubber, ['int a;', 'int c;'], ['int a;', 'int b; /* MOE:strip_line */', 'int c;'])
def assertContentsPreserved(self, contents, extractor): scrubber = comment_scrubber.CommentScrubber(extractor) fake_file = test_util.FakeFile(contents=contents) scrubber.ScrubFile(fake_file, None) self.assertMultiLineEqual(contents, fake_file.new_contents)
def testStripLineDirectiveHtml(self): scrubber = comment_scrubber.CommentScrubber( comment_scrubber.HtmlCommentExtractor()) self.assertContents( scrubber, ['a = ""', 'c = a'], ['a = ""', 'b = a <!-- MOE:strip_line -->', 'c = a'])
def testStripLineDirectiveShell(self): scrubber = comment_scrubber.CommentScrubber( comment_scrubber.ShellLikeCommentExtractor()) self.assertContents(scrubber, ['a=""', 'c=a'], ['a=""', 'b=a # MOE:strip_line', 'c=a'])
def testStartAndEndDelimitersPython(self): scrubber = comment_scrubber.CommentScrubber( comment_scrubber.PythonCommentExtractor()) self.assertContents( scrubber, ['# x', '# z'], ['# x', '# MOE:begin_strip', '# y', '# MOE:end_strip', '# z'])
def testStartAndEndDelimitersC(self): scrubber = comment_scrubber.CommentScrubber( comment_scrubber.CLikeCommentExtractor()) self.assertContents( scrubber, ['// x', '// z'], ['// x', '// MOE:begin_strip', '// y', '// MOE:end_strip', '// z'])