def test_Index_exact_match_ngrams_templates_perfect_minimalist(self):
index_doc = [u'name is joker, {{}} name is joker']
idx = index.Index(ngram_len=3)
idx.index_one('tst', text_lines(index_doc), template=True)
query_doc = [u'Hi my name is joker the joker name is joker yes.']
# 012345678901234567890123456789012345678901234567
# 11111111112222222222333333333344444444
expected = {
'tst': [(Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=33,
end=5),
Token(start=2,
start_line=0,
start_char=6,
end_line=0,
end_char=43,
end=9))]
}
matches = idx.match(text_lines(query_doc))
assert {} != matches
for k, val in matches.items():
assert expected[k] == val
def test_Index_exact_match_ngrams_perfect_minimalist(self):
index_doc = [u'name is joker, name is joker']
# 0 1 2 3 4 5
idx = index.Index(ngram_len=3)
idx.index_one('tst', text_lines(index_doc), template=False)
query_doc = [u'Hi my name is joker, name is joker yes.']
# match 0 1 |2 3 4 5 6 7| 8
expected = {
'tst': [(Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=28,
end=5),
Token(start=2,
start_line=0,
start_char=6,
end_line=0,
end_char=34,
end=7))]
}
matches = idx.match(query_doc)
assert {} != matches
for k, val in matches.items():
# assert [] == val
assert expected[k] == val
def test_Index_exact_match_ngrams_perfect_single_index_doc_in_index_minimal(
self):
test_docs = self.get_test_docs('index/mini')
idx = self.get_test_index(test_docs, ngram_len=3, template=False)
test_query_doc = self.get_test_loc('index/queryperfect-mini')
expected = {
'bsd-new': [(Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=94,
end=13),
Token(start=1,
start_line=2,
start_char=0,
end_line=2,
end_char=94,
end=14))]
}
matches = idx.match(text_lines(test_query_doc))
assert {} != matches
for k, val in matches.items():
assert expected[k] == val
def test_Index_exact_match_unigrams_perfect(self):
test_docs = self.get_test_docs('index/bsd')
idx = self.get_test_index(test_docs, ngram_len=1, template=False)
test_query_doc = self.get_test_loc('index/queryperfect')
expected = {
'bsd-new': [(Token(start=0,
start_line=0,
start_char=0,
end_line=6,
end_char=753,
end=212),
Token(start=0,
start_line=5,
start_char=0,
end_line=11,
end_char=753,
end=212))]
}
matches = idx.match(text_lines(test_query_doc))
assert {} != matches
for k, val in matches.items():
assert expected[k] == val
with codecs.open(test_query_doc, encoding='utf-8') as td:
actual = td.read().splitlines(True)
expected = u''.join(actual[5:-2])[:-2]
query_match_pos = matches['bsd-new'][0][-1]
tst = analysis.doc_subset(text_lines(location=test_query_doc),
query_match_pos)
tst = u''.join(tst)
assert expected == tst
def test_create_template_rule(self):
test_rule = models.Rule(
text_file=self.create_test_file(u'A one. A {{}}two. A three.'),
template=True)
toks = [
Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=8,
end=2,
gap=5,
value=u'a one a',
length=3),
Token(start=3,
start_line=0,
start_char=13,
end_line=0,
end_char=25,
end=5,
gap=0,
value=u'two a three',
length=3)
]
tokens, min_len, max_len, gaps_count = test_rule.get_tokens()
assert (toks, 5, 10, 5) == (tokens, min_len, max_len, gaps_count)
def test_index_template2(self):
ttr = Rule(
text_file=self.create_test_file(u'A one. A {{10}}two. A three.'),
template=True)
index = detect.LicenseIndex([ttr])
expected = {
u'a one a': {
0: [
Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=8,
end=2,
gap=10,
value=u'a one a')
]
},
u'two a three': {
0: [
Token(start=3,
start_line=0,
start_char=15,
end_line=0,
end_char=27,
end=5,
gap=0,
value=u'two a three')
]
}
}
assert expected == index.license_index.indexes[3]
def test_index_template2(self):
ttr = Rule(
text_file=self.create_test_file(u'A one. A {{10}}two. A three.'),
template=True)
index = detect.LicenseIndex([ttr])
expected = {
(u'a', u'one', u'a'): {
0: [
Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=8,
end=2,
gap=10,
value=(u'a', u'one', u'a'))
]
},
(u'two', u'a', u'three'): {
0: [
Token(start=3,
start_line=0,
start_char=15,
end_line=0,
end_char=27,
end=5,
gap=0,
value=(u'two', u'a', u'three'))
]
}
}
self.assertEqual(expected, index.license_index.indexes[3])
def test_create_plain_rule(self):
test_rule = models.Rule(
text_file=self.create_test_file('A one. A two. A three.'))
toks = [
Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=12,
end=3,
gap=0,
value=u'a one a two',
length=4),
Token(start=1,
start_line=0,
start_char=2,
end_line=0,
end_char=15,
end=4,
gap=0,
value=u'one a two a',
length=4),
Token(start=2,
start_line=0,
start_char=7,
end_line=0,
end_char=21,
end=5,
gap=0,
value=u'a two a three',
length=4),
]
tokens, min_len, max_len, gaps_count = test_rule.get_tokens()
assert (toks, 5, 5, 0) == (tokens, min_len, max_len, gaps_count)
def test_Index_exact_match_ngrams_template_perfect_multi_index_doc_in_index(
self):
test_docs = self.get_test_docs('index/bsd_templates')
idx = self.get_test_index(test_docs, ngram_len=3, template=True)
test_query_doc = self.get_test_loc(
'index/queryperfect_single_template')
expected = {
'bsd-new': [(Token(start=0,
start_line=0,
start_char=0,
end_line=6,
end_char=753,
end=210),
Token(start=4,
start_line=5,
start_char=0,
end_line=11,
end_char=753,
end=216))]
}
matches = idx.match(text_lines(test_query_doc))
assert {} != matches
for k, val in matches.items():
assert expected[k] == val
def test_Index_exact_match_to_indexed_template_with_short_tokens_around_gaps(
self):
# was failing when a gapped token (from a template) starts at a
# beginning of an index doc and at a position less than ngram length
# setup
idx = index.Index(ngram_len=4)
index_doc = text_lines(self.get_test_loc('index/templates/idx.txt'))
idx.index_one('idx', text_lines(index_doc), template=True)
# test index
quad_grams_index = idx._get_index_for_len(4)
assert 205 == len(quad_grams_index)
assert u'software without prior written' in quad_grams_index
# test match
query_doc = text_lines(self.get_test_loc('index/templates/query.txt'))
matches = idx.match(query_doc)
assert 1 == len(matches)
# we expect a single match to the idx doc
matched_query_doc_position = matches['idx'][0][1]
expected = Token(start=0,
start_line=0,
start_char=0,
end_line=39,
end_char=34,
end=276)
assert expected == matched_query_doc_position
def test_template_detection_with_short_tokens_around_gaps(self):
# failed when a gapped token starts at a beginning of rule and at a
# position less than ngram length
# setup
tf7 = self.get_test_loc('detect/templates/license7.txt')
ttr = Rule(text_file=tf7, template=True)
# use trigrams by default
index = detect.LicenseIndex([ttr])
# test the index
four_grams_index = index.license_index.indexes[3]
self.assertEqual(211, len(four_grams_index))
self.assertTrue((
u'software',
u'without',
u'prior',
) in four_grams_index)
# test
tf8 = self.get_test_loc('detect/templates/license8.txt')
matches = index.match(tf8)
self.assertEqual(1, len(matches))
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=40,
end_char=34,
end=276)
self.assertEqual(expected, matches[0].query_position)
def test_template_detection_with_short_tokens_around_gaps(self):
# failed when a gapped token starts at a beginning of rule and at a
# position less than ngram length
# setup
tf7 = self.get_test_loc('detect/templates/license7.txt')
ttr = Rule(text_file=tf7, template=True)
# use quadri grams by default
index = detect.LicenseIndex([ttr])
# test the index
quad_grams_index = index.license_index.indexes[4]
assert 205 == len(quad_grams_index)
assert u'software without prior written' in quad_grams_index
# test
tf8 = self.get_test_loc('detect/templates/license8.txt')
matches = index.match(tf8)
assert 1 == len(matches)
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=40,
end_char=34,
end=276)
assert expected == matches[0].query_position
def test_simple_detection_xcon_crlf_template(self):
# setup
tf1_text = u'''X11 License
Copyright (C) 1996 X Consortium
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Except as contained in this notice, the name of the X Consortium shall not be used in advertising or otherwise to promote the sale, use or other dealings in this Software without prior written authorization from the X Consortium.
X Window System is a trademark of X Consortium, Inc.
'''
ttr = Rule(text_file=self.create_test_file(tf1_text),
licenses=['x-consortium'],
template=True)
index = detect.LicenseIndex([ttr])
# test
doc = self.get_test_loc(
'detect/simple_detection/x11-xconsortium_text.txt')
matches = index.match(doc)
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=13,
end_char=51,
end=216)
self.assertEqual(expected, matches[0].query_position)
def test_Index_match_ngrams_templates_perfect_minimalist(self):
index_doc = [u'name is joker, {{}} name is joker']
idx = index.Index(ngram_len=3)
idx.index_one('tst', text_lines(index_doc), template=True)
query_doc = [u'Hi my name is joker the joker name is joker yes.']
expected = {
'tst': [
(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=33, end=5),
Token(start=2, start_line=0, start_char=6, end_line=0, end_char=43, end=9))
]
}
test = dict(idx.match(text_lines(query_doc), perfect=True))
self.assertNotEqual({}, test)
for k, val in test.items():
assert expected[k] == val
def test_Index_match_ngrams_template_perfect_multi_index_doc_in_index(self):
test_docs = self.get_test_docs('index/bsd_templates')
idx = self.get_test_index(test_docs, ngram_len=3, template=True)
test_query_doc = self.get_test_loc('index/queryperfect_single_template')
expected = {
'bsd-new':[
(Token(start=0, start_line=0, start_char=0, end_line=6, end_char=753, end=210),
Token(start=4, start_line=5, start_char=0, end_line=11, end_char=753, end=216))
]
}
test = dict(idx.match(text_lines(test_query_doc), perfect=True))
self.assertNotEqual({}, test)
for k, val in test.items():
assert expected[k] == val
def test_comment_format_6(self):
expected = Token(start=0,
start_line=1,
start_char=2,
end_line=9,
end_char=52,
end=86)
self.check_detection('detect/commentformat/license6.txt',
'detect/commentformat/license3.txt', expected)
def test_comment_format_4(self):
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=10,
end_char=50,
end=86)
self.check_detection('detect/commentformat/license5.txt',
'detect/commentformat/license1.txt', expected)
def load_tokens(location):
"""
Return a list of tokens loaded from a file location
"""
location = os.path.abspath(location)
with codecs.open(location, 'rb', encoding='utf-8') as loc:
loaded = loc.read().splitlines(False)
for l in loaded:
yield Token.loads(l)
def merge_positions(positions):
"""
Given a iterable of Token positions, return a new merged Token position
computed from the first and last positions (do not keep gap and token
values). Does not check if positions are contiguous or overlapping.
"""
positions = sorted(positions)
first = positions[0]
last = positions[-1]
return Token(start=first.start, end=last.end,
start_line=first.start_line, start_char=first.start_char,
end_line=last.end_line, end_char=last.end_char)
def test_merge_positions(self):
pos1 = Token(start=58,
end=58,
start_line=8,
start_char=52,
end_line=8,
end_char=59)
pos2 = Token(start=63,
end=64,
start_line=23,
start_char=12,
end_line=53,
end_char=23)
expected = Token(start=58,
end=64,
start_line=8,
start_char=52,
end_line=53,
end_char=23)
tst = index.merge_positions([pos2, pos1])
assert expected == tst
def test_create_plain_rule(self):
ftr = models.Rule(
text_file=self.create_test_file('A one. A two. A three.'))
toks = [
Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=8,
end=2,
gap=0,
value=(u'a', u'one', u'a')),
Token(start=1,
start_line=0,
start_char=2,
end_line=0,
end_char=12,
end=3,
gap=0,
value=(u'one', u'a', u'two')),
Token(start=2,
start_line=0,
start_char=7,
end_line=0,
end_char=15,
end=4,
gap=0,
value=(u'a', u'two', u'a')),
Token(start=3,
start_line=0,
start_char=9,
end_line=0,
end_char=21,
end=5,
gap=0,
value=(u'two', u'a', u'three'))
]
self.assertEqual(toks, list(ftr.get_tokens()))
for i in range(len(toks)):
self.assertEqual(toks[i], ftr.tokens[i])
def test_index_template(self):
ttr = Rule(
text_file=self.create_test_file(u'A one. A {{}}two. A three.'),
template=True)
index = detect.LicenseIndex([ttr])
expected = {
1: {},
2: {},
3: {
u'two a three': {
0: [
Token(start=3,
start_line=0,
start_char=13,
end_line=0,
end_char=25,
end=5,
gap=0,
value=u'two a three',
length=3)
]
},
u'a one a': {
0: [
Token(start=0,
start_line=0,
start_char=0,
end_line=0,
end_char=8,
end=2,
gap=5,
value=u'a one a',
length=3)
]
}
},
4: {},
}
assert expected == index.license_index.indexes
def test_Index_index_many_unigrams(self):
test_docs = self.get_test_docs('index/bsd', ['bsd-new', 'bsd-no-mod'])
idx = index.Index(ngram_len=1)
idx._index_many(test_docs)
unigrams_index = idx.indexes[1]
assert 213 == idx.get_tokens_count('bsd-new')
assert 234 == idx.get_tokens_count('bsd-no-mod')
assert 138 == len(unigrams_index)
pos = Token(start=61, end=61, start_line=8, start_char=52, end_line=8, end_char=59, value=(u'minimum',))
expected_posting = ('bsd-no-mod', [pos],)
assert expected_posting == unigrams_index[('minimum',)].items()[0]
def test_bsd_rule_detection(self):
tf1 = self.get_test_loc('detect/mit/t1.txt')
ftr = Rule(text_file=tf1, licenses=['bsd-original'])
index = detect.LicenseIndex([ftr])
test_doc = self.get_test_loc('detect/mit/t2.txt')
matches = index.match(test_doc)
self.assertEqual(1, len(matches))
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=27,
end_char=59,
end=241)
self.assertEqual(expected, matches[0].query_position)
def test_fulltext_detection_works_with_partial_overlap_from_location(self):
# setup
test_rule = self.get_test_loc('detect/templates/license3.txt')
ftr = Rule(text_file=test_rule, licenses=['mylicense'])
index = detect.LicenseIndex([ftr])
# test
test_doc = self.get_test_loc('detect/templates/license4.txt')
matches = index.match(test_doc)
self.assertEqual(1, len(matches))
expected = Token(start=1,
start_line=1,
start_char=7,
end_line=4,
end_char=67,
end=42)
self.assertEqual(expected, matches[0].query_position)
def test_template_detection_publicdomain(self):
# setup
tf5 = self.get_test_loc('detect/templates/license5.txt')
ttr = Rule(text_file=tf5, licenses=['public-domain'], template=True)
index = detect.LicenseIndex([ttr])
# test
tf6 = self.get_test_loc('detect/templates/license6.txt')
matches = index.match(tf6)
self.assertEqual(1, len(matches))
expected = Token(start=82,
start_line=16,
start_char=0,
end_line=18,
end_char=67,
end=118)
self.assertEqual(expected, matches[0].query_position)
def test_merge_aligned_positions(self):
pos11 = Token(start=58, end=58, start_line=8, start_char=52, end_line=8, end_char=59)
pos12 = Token(start=63, end=64, start_line=23, start_char=12, end_line=53, end_char=23)
pos21 = Token(start=12, end=24, start_line=12, start_char=52, end_line=12, end_char=59)
pos22 = Token(start=15, end=35, start_line=45, start_char=12, end_line=54, end_char=36)
expected = (
Token(start=58, end=64, start_line=8, start_char=52, end_line=53, end_char=23),
Token(start=12, end=35, start_line=12, start_char=52, end_line=54, end_char=36)
)
tst = index.merge_aligned_positions([(pos11, pos21,), (pos12, pos22,)])
assert expected == tst
def test_Index_index_several_unigrams(self):
test_docs = self.get_test_docs('index/bsd', ['bsd-new', 'bsd-no-mod'])
idx = index.Index(ngram_len=1)
for docid, doc in test_docs:
idx.index_one(docid, doc)
unigrams_index = idx.indexes[1]
assert 213 == idx.get_tokens_count('bsd-new')
assert 234 == idx.get_tokens_count('bsd-no-mod')
assert 138 == len(unigrams_index)
pos = Token(start=61,
end=61,
start_line=8,
start_char=52,
end_line=8,
end_char=59,
value=u'minimum')
expected_posting = (
'bsd-no-mod',
[pos],
)
assert expected_posting == unigrams_index['minimum'].items()[0]
def test_detection_template_with_inter_gap_smaller_than_ngram_len(self):
# in this template text there are only 2 tokens between the two
# templates: this is smaller than the ngram_len of 3 and can never be
# caught by this length
tf1_text = u'''Redistributions in binary form must
{{}} reproduce the {{}}above copyright notice'''
ttr = Rule(text_file=self.create_test_file(tf1_text),
licenses=['mylicense'],
template=True)
index = detect.LicenseIndex([ttr]) # default to ngram_len=3
# test
tf2 = u'''Redistributions in binary form must nexB company
reproduce the word for word above copyright notice.'''.splitlines()
matches = index.match(tf2)
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=2,
end_char=58,
end=14)
self.assertEqual(1, len(matches))
self.assertEqual(expected, matches[0].query_position)
def test_detection_template_with_inter_gap_bigger_than_ngram_len(self):
# setup in this template there are only 4 tokens between the two
# templates: this is bigger than the ngram_len of 3
tf1_text = u'''Redistributions in binary form must
{{}} reproduce as is stipulated {{}}above copyright notice'''
ttr = Rule(text_file=self.create_test_file(tf1_text),
licenses=['mylicense'],
template=True)
index = detect.LicenseIndex([ttr]) # default to ngram_len=3
# test
tf2_text = (u'''Redistributions in binary form must nexB company
reproduce as is stipulated the word for word above copyright notice.'''
.splitlines())
matches = index.match(tf2_text)
expected = Token(start=0,
start_line=1,
start_char=0,
end_line=2,
end_char=75,
end=17)
self.assertEqual(1, len(matches))
self.assertEqual(expected, matches[0].query_position)
