def _MakeShellScrubbers(self):
shell_scrubbers = []
shell_scrubbers.append(
comment_scrubber.CommentScrubber(
comment_scrubber.ShellLikeCommentExtractor(),
comment_scrubbers=self._CommentScrubbers()))
shell_scrubbers.extend(self._PolyglotFileScrubbers())
return shell_scrubbers
def assertContentsError(self, lines):
context = FakeContext()
scrubber = comment_scrubber.CommentScrubber(
comment_scrubber.CLikeCommentExtractor())
file_obj = test_util.FakeFile(contents=u'\n'.join(lines))
scrubber.ScrubFile(file_obj, context)
self.assertEqual(1, len(context.errors))
error, = context.errors
self.assertEqual('COMMENT_SCRUBBER_ERROR', error.filter)
def _MakeHtmlScrubbers(self):
html_scrubbers = []
html_scrubbers.append(
comment_scrubber.CommentScrubber(
comment_scrubber.HtmlCommentExtractor(),
self._CommentScrubbers()))
line_scrubbers = self._PolyglotLineOrientedScrubbers()
for js_directory_rename in self.js_directory_renames:
line_scrubbers.append(js_directory_rename)
html_scrubbers.append(line_scrubber.LineScrubber(line_scrubbers))
html_scrubbers.extend(self._PolyglotFileScrubbers())
return html_scrubbers
def RunScenario(self, scenario_name, extractor=None):
if not extractor:
extractor = comment_scrubber.CLikeCommentExtractor()
scrubber = comment_scrubber.CommentScrubber(extractor)
scenario_dir = os.path.join(TEST_DATA_DIR, 'directives', scenario_name)
unstripped = os.path.join(scenario_dir, 'input.txt')
file_obj = test_util.FakeFile(filename=unstripped)
scrubber.ScrubFile(file_obj, None)
expected = os.path.join(scenario_dir, 'expected.txt')
out = os.path.join(FLAGS.test_tmpdir, scenario_name + '.out.txt')
file_util.Write(out, file_obj.new_contents)
basetest.DiffTestFiles(out, expected)
def __init__(self, action):
"""Initialize.
Args:
action: base.ACTION_* constant, what to do to an unmeaningful file.
"""
base.FileScrubber.__init__(self)
# Use a separate extractor and scrubber. We don't want the scrubber to
# actually scrub, but we need to extract comments to pass to
# _comment_scrubber.DetermineNewContents.
self._extractor = comment_scrubber.CLikeCommentExtractor()
self._comment_scrubber = comment_scrubber.CommentScrubber(
extractor=None, comment_scrubbers=[RemoveCommentsScrubber()])
self._action = action
def _ResetBatchScrubbers(self):
"""Set batch scrubbers to run before and after by-file scrubbing.
See ScrubberContext.Scan below. First, the pre-batch scrubbers are run for
applicable files, then by-file scrubbers, then post-batch scrubbers.
("Pre-batch" and "post-batch" are misnomers, but used for simplicity.
Respectively, they're really "pre-(by-file)" and "post-(by-file)".)
"""
c_like_comment_pre_batch_scrubbers = [
comment_scrubber.CommentScrubber(
comment_scrubber.CLikeCommentExtractor(),
self._CommentScrubbers())
]
java_pre_batch_scrubbers = []
java_pre_batch_scrubbers.extend(c_like_comment_pre_batch_scrubbers)
proto_pre_batch_scrubbers = (self.scrub_proto_comments
and c_like_comment_pre_batch_scrubbers
or [])
self.extension_to_pre_batch_scrubbers_map = {
'.c': c_like_comment_pre_batch_scrubbers,
'.cc': c_like_comment_pre_batch_scrubbers,
'.go': c_like_comment_pre_batch_scrubbers,
'.h': c_like_comment_pre_batch_scrubbers,
'.java': java_pre_batch_scrubbers,
'.jj': java_pre_batch_scrubbers,
'.js': c_like_comment_pre_batch_scrubbers,
'.jslib': c_like_comment_pre_batch_scrubbers,
'.l': c_like_comment_pre_batch_scrubbers,
'.php': c_like_comment_pre_batch_scrubbers,
'.php4': c_like_comment_pre_batch_scrubbers,
'.php5': c_like_comment_pre_batch_scrubbers,
'.proto': proto_pre_batch_scrubbers,
'.protodevel': proto_pre_batch_scrubbers,
'.swig': c_like_comment_pre_batch_scrubbers,
}
java_post_batch_scrubbers = []
if self.empty_java_file_action != base.ACTION_IGNORE:
java_post_batch_scrubbers.append(
java_scrubber.EmptyJavaFileScrubber(
self.empty_java_file_action))
self.extension_to_post_batch_scrubbers_map = {
'.java': java_post_batch_scrubbers,
'.jj': java_post_batch_scrubbers,
}
def _MakePythonScrubbers(self):
py_scrubbers = []
py_scrubbers.append(
comment_scrubber.CommentScrubber(
comment_scrubber.PythonCommentExtractor(),
self._CommentScrubbers()))
line_scrubbers = []
line_scrubbers.extend(self.python_module_renames)
line_scrubbers.extend(self.python_module_removes)
if self.scrub_authors:
line_scrubbers.append(
line_scrubber.PythonAuthorDeclarationScrubber(
self.username_filter))
if self.python_shebang_replace:
py_scrubbers.append(self.python_shebang_replace)
line_scrubbers += self._PolyglotLineOrientedScrubbers()
py_scrubbers.append(line_scrubber.LineScrubber(line_scrubbers))
py_scrubbers.extend(self._PolyglotFileScrubbers())
return py_scrubbers
def testStripLineDirectiveC(self):
scrubber = comment_scrubber.CommentScrubber(
comment_scrubber.CLikeCommentExtractor())
self.assertContents(scrubber, ['int a;', 'int c;'],
['int a;', '// MOE:strip_line', 'int c;'])
self.assertContents(scrubber, ['int a;', 'int c;'], [
'int a;',
'int b; // MOE:strip_line The preceding relies on the magic seed 42',
'int c;'
])
self.assertContents(scrubber, ['int a;', 'int c;'],
['int a;', 'int b; // MOE:strip_line', 'int c;'])
self.assertContents(scrubber, ['int a;', 'int c;'], [
'int a;', 'int b1; // MOE:strip_line',
'int b2; // MOE:strip_line', 'int c;'
])
self.assertContents(scrubber, ['int a;', 'int c;'], [
'int a;',
'int b; // This line and comment are secret. MOE:strip_line',
'int c;'
])
self.assertContents(
scrubber, ['int a;', 'int c;'],
['int a;', 'int b; /* MOE:strip_line */', 'int c;'])
def assertContentsPreserved(self, contents, extractor):
scrubber = comment_scrubber.CommentScrubber(extractor)
fake_file = test_util.FakeFile(contents=contents)
scrubber.ScrubFile(fake_file, None)
self.assertMultiLineEqual(contents, fake_file.new_contents)
def testStripLineDirectiveHtml(self):
scrubber = comment_scrubber.CommentScrubber(
comment_scrubber.HtmlCommentExtractor())
self.assertContents(
scrubber, ['a = ""', 'c = a'],
['a = ""', 'b = a <!-- MOE:strip_line -->', 'c = a'])
def testStripLineDirectiveShell(self):
scrubber = comment_scrubber.CommentScrubber(
comment_scrubber.ShellLikeCommentExtractor())
self.assertContents(scrubber, ['a=""', 'c=a'],
['a=""', 'b=a # MOE:strip_line', 'c=a'])
def testStartAndEndDelimitersPython(self):
scrubber = comment_scrubber.CommentScrubber(
comment_scrubber.PythonCommentExtractor())
self.assertContents(
scrubber, ['# x', '# z'],
['# x', '# MOE:begin_strip', '# y', '# MOE:end_strip', '# z'])
def testStartAndEndDelimitersC(self):
scrubber = comment_scrubber.CommentScrubber(
comment_scrubber.CLikeCommentExtractor())
self.assertContents(
scrubber, ['// x', '// z'],
['// x', '// MOE:begin_strip', '// y', '// MOE:end_strip', '// z'])