def test_given_file_with_a_line_when_empty_pattern_then_prints_line(capsys, fileutil):
path = fileutil.create_file('oneline', 'one line')
grep('', path)
out, err = capsys.readouterr()
assert 'one line' in out
def test_given_empty_lines_when_grepping_does_not_print_additional_lines(capsys, fileutil):
path = fileutil.create_file('empty_lines', 'a\n\na\n')
grep('a', path)
out, err = capsys.readouterr()
assert out == 'a\na\n'
def test_given_empty_file_when_empty_pattern_then_prints_nothing(capsys, fileutil):
path = fileutil.create_file('emptyfile', '')
grep('', path)
out, err = capsys.readouterr()
assert out == "\n"
def test_given_file_when_using_regex_then_prints_matching_lines(capsys, fileutil):
path = fileutil.create_file('file', 'a\n1\nb\n2\n')
grep('[12]', path)
out, err = capsys.readouterr()
assert '1' in out
assert '2' in out
def test_given_file_with_two_lines_when_pattern_contains_a_line_then_prints_line(capsys, fileutil):
path = fileutil.create_file('twolines', 'a\nb\n')
grep('b', path)
out, err = capsys.readouterr()
assert 'b' in out
assert 'a' not in out
def test_given_option_ignore_case_when_matching_capital_letters_then_prints_matching_lines(capsys, fileutil):
path = fileutil.create_file('uppercase', 'A\nb\n')
grep('a|b', path, ignore_case=True)
out, err = capsys.readouterr()
assert 'A' in out
assert 'b' in out
def test_given_option_print_line_number_when_grepping_then_prints_line_numbers(capsys, fileutil):
path = fileutil.create_file('line_numbers', 'a\nb\naa\nbb\n')
grep('a', path, line_number=True)
out, err = capsys.readouterr()
assert '1:a' in out
assert '3:aa' in out
def test_given_2_files_when_grepping_then_prints_lines_in_all_files(capsys, fileutil):
first_file = fileutil.create_file('first', 'aa\nbb\n')
second_file = fileutil.create_file('second', 'ab\nbc\n')
grep('a', first_file, second_file)
out, err = capsys.readouterr()
assert 'aa' in out
assert 'ab' in out
def test_one_file_several_matches_print_line_numbers_flag(self):
self.assertMultiLineEqual(
grep("may", [MIDSUMMERNIGHTFILENAME], "-n"),
"3:Nor how it may concern my modesty,\n"
"5:But I beseech your grace that I may know\n"
"6:The worst that may befall me in this case,\n"
)
def test_multiple_files_several_matches_inverted_flag(self):
self.assertMultiLineEqual(
grep("a", FILENAMES, "-v"),
"iliad.txt:Achilles sing, O Goddess! Peleus' son;\n"
"iliad.txt:The noble Chief Achilles from the son\n"
"midsummer-night.txt:If I refuse to wed Demetrius.\n"
)
def test_multiple_files_several_matches_no_flags(self):
self.assertMultiLineEqual(
grep("may", FILENAMES),
"midsummer-night.txt:Nor how it may concern my modesty,\n"
"midsummer-night.txt:But I beseech your grace that I may know\n"
"midsummer-night.txt:The worst that may befall me in this case,\n"
)
def regexp_category_line(file_name, word):
text_list = grep.grep(file_name, word)
p = re.compile('Category') # TODO: media wiki 記法
for text in text_list:
for line in text.split("\n"):
if p.findall(line): # TODO: search? match? でできないか
print(line, end = "\n")
def test_multiple_files_several_matches_print_line_numbers_flag(self):
expected = (
"midsummer-night.txt:5:But I beseech your grace that I may know\n"
"midsummer-night.txt:6:The worst that may befall me in this case,"
"\nparadise-lost.txt:2:Of that Forbidden Tree, whose mortal tast\n"
"paradise-lost.txt:6:Sing Heav'nly Muse, that on the secret top\n")
self.assertMultiLineEqual(grep("that", "-n", FILENAMES), expected)
def test_multiple_files_several_matches_inverted_match_entire_lines(self):
expected = (
"iliad.txt:Achilles sing, O Goddess! Peleus' son;\n"
"iliad.txt:His wrath pernicious, who ten thousand woes\n"
"iliad.txt:Caused to Achaia's host, sent many a soul\n"
"iliad.txt:And Heroes gave (so stood the will of Jove)\n"
"iliad.txt:To dogs and to all ravening fowls a prey,\n"
"iliad.txt:When fierce dispute had separated once\n"
"iliad.txt:The noble Chief Achilles from the son\n"
"iliad.txt:Of Atreus, Agamemnon, King of men.\n"
"midsummer-night.txt:I do entreat your grace to pardon me.\n"
"midsummer-night.txt:I know not by what power I am made bold,\n"
"midsummer-night.txt:Nor how it may concern my modesty,\n"
"midsummer-night.txt:In such a presence here to plead my thoughts;"
"\nmidsummer-night.txt:But I beseech your grace that I may know\n"
"midsummer-night.txt:The worst that may befall me in this case,\n"
"midsummer-night.txt:If I refuse to wed Demetrius.\n"
"paradise-lost.txt:Of Mans First Disobedience, and the Fruit\n"
"paradise-lost.txt:Of that Forbidden Tree, whose mortal tast\n"
"paradise-lost.txt:Brought Death into the World, and all our woe,"
"\nparadise-lost.txt:With loss of Eden, till one greater Man\n"
"paradise-lost.txt:Restore us, and regain the blissful Seat,\n"
"paradise-lost.txt:Sing Heav'nly Muse, that on the secret top\n"
"paradise-lost.txt:Of Oreb, or of Sinai, didst inspire\n"
"paradise-lost.txt:That Shepherd, who first taught the chosen Seed"
"\n"
)
self.assertMultiLineEqual(
grep("Illustrious into Ades premature,", "-x -v", FILENAMES),
expected
)
def test_one_file_several_matches_no_flags(self):
self.assertMultiLineEqual(
grep("may", [MIDSUMMERNIGHTFILENAME]),
"Nor how it may concern my modesty,\n"
"But I beseech your grace that I may know\n"
"The worst that may befall me in this case,\n"
)
def regexp_section(file_name, word):
text_list = grep.grep(file_name, word)
for text in text_list:
for line in text.split("\n"):
m = re.findall('(^=+)(.*?)=+$', line)
if m:
print(m[0][1] + " " + str(len(m[0][0])))
def test_one_file_several_matches_inverted_flag(self):
self.assertMultiLineEqual(
grep("Of", "-v", [PARADISELOSTFILENAME]),
"Brought Death into the World, and all our woe,\n"
"With loss of Eden, till one greater Man\n"
"Restore us, and regain the blissful Seat,\n"
"Sing Heav'nly Muse, that on the secret top\n"
"That Shepherd, who first taught the chosen Seed\n")
def test_one_file_several_matches_inverted_flag(self):
self.assertMultiLineEqual(
grep("Of", "-v", [PARADISELOSTFILENAME]),
"Brought Death into the World, and all our woe,\n"
"With loss of Eden, till one greater Man\n"
"Restore us, and regain the blissful Seat,\n"
"Sing Heav'nly Muse, that on the secret top\n"
"That Shepherd, who first taught the chosen Seed\n")
def main(pattern, base, prime):
text = load_text()
# text = "bcdafdgklm"
start_time = time.time()
# print(grep(256, pattern, text, 2097211))
print(grep(pattern, text, base, prime))
# print(grep_brute(pattern, text))
print("--- %s seconds ---" % (time.time() - start_time))
def test_multiple_files_one_match_multiple_flags(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
"WITH LOSS OF EDEN, TILL ONE GREATER MAN",
"-n -i -x",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"paradise-lost.txt:4:With loss of Eden, till one greater Man\n",
)
def test_one_file_one_match_multiple_flags(self):
self.assertMultiLineEqual(
grep(
"OF ATREUS, Agamemnon, KIng of MEN.",
[ILIADFILENAME],
"-n -i -x"
),
"9:Of Atreus, Agamemnon, King of men.\n"
)
def regexp_file(file_name, word):
text_list = grep.grep(file_name, word)
list = [];
for text in text_list:
for line in text.split("\n"):
m = re.findall('ファイル:(.*?)\|', line)
if m:
list = list + m
return list
def test_multiple_files_several_matches_print_line_numbers_flag(self):
self.assertMultiLineEqual(
grep("that", "-n",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"]),
"midsummer-night.txt:5:But I beseech your grace that I may know\n"
"midsummer-night.txt:6:The worst that may befall me in this case,\n"
"paradise-lost.txt:2:Of that Forbidden Tree, whose mortal tast\n"
"paradise-lost.txt:6:Sing Heav'nly Muse, that on the secret top\n",
)
def regexp_category(file_name, word):
text_list = grep.grep(file_name, word)
list = []
for text in text_list:
for line in text.split("\n"):
m = re.fullmatch('\[\[Category:((.*?)(\|.*?)?)\]\]', line)
if m:
list.append(m.group(2))
return list
def test_multiple_files_one_match_match_entire_lines_flag(self):
self.assertMultiLineEqual(
grep(
"But I beseech your grace that I may know",
"-x",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"midsummer-night.txt:But I beseech your grace that I may know\n",
)
def test_multiple_files_several_matches_no_flags(self, mock_file,
mock_open):
self.assertMultiLineEqual(
grep("may", "",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"]),
"midsummer-night.txt:Nor how it may concern my modesty,\n"
"midsummer-night.txt:But I beseech your grace that I may know\n"
"midsummer-night.txt:The worst that may befall me in this case,\n",
)
def test_multiple_files_one_match_no_flags(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
"Agamemnon",
"",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"iliad.txt:Of Atreus, Agamemnon, King of men.\n",
)
def test_multiple_files_several_matches_inverted_flag(
self, mock_file, mock_open):
self.assertMultiLineEqual(
grep("a", "-v",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"]),
"iliad.txt:Achilles sing, O Goddess! Peleus' son;\n"
"iliad.txt:The noble Chief Achilles from the son\n"
"midsummer-night.txt:If I refuse to wed Demetrius.\n",
)
def test_one_file_several_matches_print_line_numbers_flag(
self, mock_file, mock_open
):
self.assertMultiLineEqual(
grep("may", "-n", ["midsummer-night.txt"]),
"3:Nor how it may concern my modesty,\n"
"5:But I beseech your grace that I may know\n"
"6:The worst that may befall me in this case,\n",
)
def test_multiple_files_no_matches_various_flags(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
"Frodo",
"-n -l -x -i",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"",
)
def category_name(file_name, word):
text_list = grep.grep(file_name, word)
dist = {}
for text in text_list:
for line in text.split("\n"):
p = re.compile('\A\|(.*?) = (.*?)\Z'); # TODO fix
m = p.match(line)
if m != None:
dist[m.group(1)] = m.group(2)
return dist
def test_regexp_search_word_starting_with_lowercase_letter(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
r"\bw[\S]+\b", # word starting with lowercase letter w
"-e",
["iliad.txt"]
),
"His wrath pernicious, who ten thousand woes\n"
"And Heroes gave (so stood the will of Jove)\n"
)
def test_regexp_search_word_how_who(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
r"\bhow\b|\bwho\b",
"-e -n",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"iliad.txt:2:His wrath pernicious, who ten thousand woes\n"
"midsummer-night.txt:3:Nor how it may concern my modesty,\n"
"paradise-lost.txt:8:That Shepherd, who first taught the chosen Seed\n"
)
def test_multiple_files_several_matches_file_flag_takes_precedence_over_line_number_flag(
self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
"who",
"-n -l",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"iliad.txt\n"
"paradise-lost.txt\n",
)
def test_regexp_search_word_how_who_prefix_file_only(self, mock_file, mock_open):
# using who as prefix => match whose
self.assertMultiLineEqual(
grep(
r"\bhow\b|\bwho\S*\b",
"-e -n -l",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"iliad.txt\n"
"midsummer-night.txt\n"
"paradise-lost.txt\n",
)
def add():
sys.stdout.write("enter new word: ")
sys.stdout.flush()
word = sys.stdin.readline().strip()
content = grep(word)
if content:
sys.stdout.write("found the following translation:\n")
line()
sys.stdout.write(content)
line()
else:
sys.stdout.write("not translation found\n")
level = find(word)
if level:
sys.stdout.write("word already exists in level %d\n" % level)
display(level, word)
def merge():
if content:
file = getFile(level, word)
f = open(file, "a")
f.write(content)
f.close()
edit(level, word)
def menu():
return Menu("what to do?", (
("edit", "e", partial(edit, level, word)),
("move to level %d" % Config.minLevel, "m", partial(setLevel, level, word, Config.minLevel)),
("merge translations", "g", merge),
))
play(menu)
else:
def create(doEdit):
file = getFile(Config.minLevel, word)
f = open(file, "w")
f.write(content)
f.close()
if doEdit:
edit(Config.minLevel, word)
return True
def menu():
return Menu("what to do?", (
("add word", "a", partial(create, False)),
("add word and edit", "e", partial(create, True)),
))
play(menu)
def test_multiple_files_several_matches_case_insensitive_flag(self):
expected = (
"iliad.txt:Caused to Achaia's host, sent many a soul\n"
"iliad.txt:Illustrious into Ades premature,\n"
"iliad.txt:And Heroes gave (so stood the will of Jove)\n"
"iliad.txt:To dogs and to all ravening fowls a prey,\n"
"midsummer-night.txt:I do entreat your grace to pardon me.\n"
"midsummer-night.txt:In such a presence here to plead my thoughts;"
"\nmidsummer-night.txt:If I refuse to wed Demetrius.\n"
"paradise-lost.txt:Brought Death into the World, and all our woe,"
"\nparadise-lost.txt:Restore us, and regain the blissful Seat,\n"
"paradise-lost.txt:Sing Heav'nly Muse, that on the secret top\n")
self.assertMultiLineEqual(grep("TO", "-i", FILENAMES), expected)
def test_regexp_search_word_how_who_prefix(self, mock_file, mock_open):
# using who as prefix => match whose
self.assertMultiLineEqual(
grep(
r"\bhow\b|\bwho\S*\b",
"-e -n",
["iliad.txt", "midsummer-night.txt", "paradise-lost.txt"],
),
"iliad.txt:2:His wrath pernicious, who ten thousand woes\n"
"midsummer-night.txt:3:Nor how it may concern my modesty,\n"
"paradise-lost.txt:2:Of that Forbidden Tree, whose mortal tast\n"
"paradise-lost.txt:8:That Shepherd, who first taught the chosen Seed\n"
)
def test_multiple_files_several_matches_case_insensitive_flag(self):
expected = (
"iliad.txt:Caused to Achaia's host, sent many a soul\n"
"iliad.txt:Illustrious into Ades premature,\n"
"iliad.txt:And Heroes gave (so stood the will of Jove)\n"
"iliad.txt:To dogs and to all ravening fowls a prey,\n"
"midsummer-night.txt:I do entreat your grace to pardon me.\n"
"midsummer-night.txt:In such a presence here to plead my thoughts;"
"\nmidsummer-night.txt:If I refuse to wed Demetrius.\n"
"paradise-lost.txt:Brought Death into the World, and all our woe,"
"\nparadise-lost.txt:Restore us, and regain the blissful Seat,\n"
"paradise-lost.txt:Sing Heav'nly Muse, that on the secret top\n")
self.assertMultiLineEqual(grep("TO", "-i", FILENAMES), expected)
def test_one_file_several_matches_inverted_and_match_entire_lines_flags(
self, mock_file, mock_open):
self.assertMultiLineEqual(
grep("Illustrious into Ades premature,", "-x -v", ["iliad.txt"]),
"Achilles sing, O Goddess! Peleus' son;\n"
"His wrath pernicious, who ten thousand woes\n"
"Caused to Achaia's host, sent many a soul\n"
"And Heroes gave (so stood the will of Jove)\n"
"To dogs and to all ravening fowls a prey,\n"
"When fierce dispute had separated once\n"
"The noble Chief Achilles from the son\n"
"Of Atreus, Agamemnon, King of men.\n",
)
def test_regexp_search_word_starting_with_A(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep(
r"\bA[\S]+\b", # word starting with uppercase A
"-e",
["iliad.txt"] # , "midsummer-night.txt", "paradise-lost.txt"],
),
"Achilles sing, O Goddess! Peleus' son;\n"
"Caused to Achaia's host, sent many a soul\n"
"Illustrious into Ades premature,\n"
"And Heroes gave (so stood the will of Jove)\n"
"The noble Chief Achilles from the son\n"
"Of Atreus, Agamemnon, King of men.\n"
)
def main():
#paths = glob.glob("DDRS/ddrs/(0-9)*/*")
parser = argparse.ArgumentParser(description='Description of your program')
parser.add_argument('pickled_paths')
parser.add_argument('grep_out_path')
parser.add_argument('eo_out_path')
parser.add_argument('meta_table_path')
args = vars(parser.parse_args())
paths = pickle.load(open('../data/paths.pickle', 'r'))
#paths = glob.glob("../data/ddrs/*/*")
with open(args['grep_out_path'], 'w') as grep_out:
for line in grep(paths, "E.O."):
grep_out.write(line)
## regex patterns ###
path_patt = re.compile('../data/ddrs/(\d+)/(\d+)\.txt')
text_patt = re.compile('E\.O\.\s*(.*)[\s\.,]*(.*)')
eo_patt = re.compile('.*E\.O\.\s*(\d*).*')
sec_patt = re.compile('.*(sec|sac)\.*\s*(.*)', re.IGNORECASE)
# write cleaned tables to file
with open(args['eo_out_path'], 'w') as fout:
fout.write("number\tpath\tdoc_id\tpage\teo_id\tsec\n")
i = 0
for line in open(args['grep_out_path'], 'r'):
m = path_patt.match(line)
path = m.group()
doc_id = m.group(1)
page = m.group(2)
m = eo_patt.match(line)
eo_id = ""
if m:
eo_id = m.group(1)
m = sec_patt.match(line)
sec = ""
if m:
sec = m.group(2)
fout.write(str(i) + "\t" + path + "\t" + doc_id + "\t" + page + "\t" + eo_id + "\t" + sec + "\n")
i += 1
def search(self, args):
"""::help::
search logs matches the kewword.
example:
>>> search CREATE,ISDIR
>>> 12:00 CREATE LOG1(too long too show you in help)
>>> 12:01 DELETE LOG2
:param args: keyword in log
"""
if self.log_path == '':
print 'Set path first!!!'
return
if len(args) == 0:
print 'Plearse enter the keyword!!!'
return
keyword = '\b'.join(args)
for log in result_parser(self.regex, self.labels,
grep(self.log_path, keyword)):
print (' '*5).join(log)
def search(self, args):
"""::help::
search logs matches the kewword.
example:
>>> search CREATE,ISDIR
>>> 12:00 CREATE LOG1(too long too show you in help)
>>> 12:01 DELETE LOG2
:param args: keyword in log
"""
if self.log_path == '':
print 'Set path first!!!'
return
if len(args) == 0:
print 'Plearse enter the keyword!!!'
return
keyword = '\b'.join(args)
for log in result_parser(self.regex, self.labels,
grep(self.log_path, keyword)):
print(' ' * 5).join(log)
def extractHooks(self):
matches = grep(self._path, "^.*<(inject-(.*))>.*$")
hooks = []
sources = self.objdump.getSources()
for path, lineno, match in matches:
fname = os.path.basename(path)
line = None
for other in sources:
if fname == os.path.basename(other):
line = sources[other][lineno]
break
if line:
hooks.append((line, match.group(2), match.group(0)))
else:
print "Couldn't match hook (%s)" % (matches.group(0))
self._hooks = hooks
return hooks
def test_multiple_files_no_matches_various_flags(self):
self.assertMultiLineEqual(
grep("Frodo", FILENAMES, "-n -l -x -i"),
""
)
import grep
import glob
grep.grep("\<rather\>", glob.glob("samples/*.txt"))
# 4: indentation, rather than delimiters, might become
def test_one_file_one_match_print_file_names_flag(self, mock_file,
mock_open):
self.assertMultiLineEqual(
grep("Forbidden", "-l", ["paradise-lost.txt"]),
"paradise-lost.txt\n")
def test_one_file_one_match_print_line_numbers_flag(
self, mock_file, mock_open):
self.assertMultiLineEqual(
grep("Forbidden", "-n", ["paradise-lost.txt"]),
"2:Of that Forbidden Tree, whose mortal tast\n",
)
def test_multiple_files_several_matches_file_flag_takes_precedence(self):
self.assertMultiLineEqual(
grep("who", "-n -l", FILENAMES),
ILIADFILENAME + '\n' + PARADISELOSTFILENAME + '\n')
def test_multiple_files_no_matches_various_flags(self):
self.assertMultiLineEqual(grep("Frodo", "-n -l -x -i", FILENAMES), "")
def test_multiple_files_one_match_multiple_flags(self):
self.assertMultiLineEqual(
grep("WITH LOSS OF EDEN, TILL ONE GREATER MAN", "-n -i -x",
FILENAMES),
"paradise-lost.txt:4:With loss of Eden, till one greater Man\n")
def test_multiple_files_one_match_match_entire_lines_flag(self):
self.assertMultiLineEqual(
grep("But I beseech your grace that I may know",
FILENAMES, "-x"),
"midsummer-night.txt:But I beseech your grace that I may know\n")
def test_multiple_files_one_match_multiple_flags(self):
self.assertMultiLineEqual(
grep("WITH LOSS OF EDEN, TILL ONE GREATER MAN",
FILENAMES, "-n -i -x"),
"paradise-lost.txt:4:With loss of Eden, till one greater Man\n")
def test_one_file_one_match_no_flags(self, mock_file, mock_open):
self.assertMultiLineEqual(grep("Agamemnon", "", ["iliad.txt"]),
"Of Atreus, Agamemnon, King of men.\n")
def test_one_file_several_matches_match_entire_lines_flag(
self, mock_file, mock_open):
self.assertMultiLineEqual(grep("may", "-x", ["midsummer-night.txt"]),
"")
def test_one_file_one_match_case_insensitive_flag(self, mock_file,
mock_open):
self.assertMultiLineEqual(
grep("FORBIDDEN", "-i", ["paradise-lost.txt"]),
"Of that Forbidden Tree, whose mortal tast\n",
)
def test_one_file_no_matches_various_flags(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep("Gandalf", "-n -l -x -i", ["iliad.txt"]), "")
def test_one_file_one_match_multiple_flags(self, mock_file, mock_open):
self.assertMultiLineEqual(
grep("OF ATREUS, Agamemnon, KIng of MEN.", "-n -i -x",
["iliad.txt"]),
"9:Of Atreus, Agamemnon, King of men.\n",
)
def test_one_file_one_match_file_flag_takes_precedence_over_line_flag(
self, mock_file, mock_open):
self.assertMultiLineEqual(grep("ten", "-n -l", ["iliad.txt"]),
"iliad.txt\n")
