def test_Query_tokenize_from_string(self):
rule_text = 'Redistribution and use in source and binary forms with or without modification are permitted'
idx = index.LicenseIndex([Rule(stored_text=rule_text, license_expression='bsd')])
querys = '''
The
Redistribution and use in source and binary are permitted.
Athena capital of Grece
Paris and Athene
Always'''
qry = Query(query_string=querys, idx=idx, _test_mode=True)
tokens_by_line = list(qry.tokens_by_line(query_string=querys))
qry.tokenize_and_build_runs(tokens_by_line)
expected = ['redistribution', 'and', 'use', 'in', 'source', 'and', 'binary', 'are', 'permitted', 'and']
result = tks_as_str(qry.tokens, idx=idx)
assert result == expected
expected = [None, 'redistribution', 'and', 'use', 'in', 'source', 'and', 'binary', 'are', 'permitted', None, None, None, None, None, 'and', None, None]
result = tks_as_str(query_tokens_with_unknowns(qry), idx=idx)
assert result == expected
assert len(qry.query_runs) == 1
qr1 = qry.query_runs[0]
assert qr1.start == 0
assert qr1.end == 9
assert len(qr1) == 10
expected = ['redistribution', 'and', 'use', 'in', 'source', 'and', 'binary', 'are', 'permitted', 'and']
result = tks_as_str(qr1.tokens, idx=idx)
assert result == expected
expected = [None, 'redistribution', 'and', 'use', 'in', 'source', 'and', 'binary', 'are', 'permitted', None, None, None, None, None, 'and']
result = tks_as_str(query_run_tokens_with_unknowns(qr1), idx=idx)
assert result == expected
def test_match_with_templates_with_redundant_tokens_yield_single_exact_match(self):
from licensedcode_test_utils import query_tokens_with_unknowns # NOQA
_stored_text = 'copyright reserved mit is license, copyright reserved mit is license'
# 0 1 2 3 4 5 6 7 8 9
license_expression = 'tst'
rule = models.Rule(license_expression=license_expression, stored_text=_stored_text)
idx = MiniLicenseIndex([rule])
querys = u'Hi my copyright reserved mit is license is the copyright reserved mit is license yes.'
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
qry = Query(query_string=querys, idx=idx)
# convert tid to actual token strings
tks_as_str = lambda tks: [None if tid is None else idx.tokens_by_tid[tid] for tid in tks]
expected = [None, None, u'copyright', u'reserved', u'mit', u'is', u'license', u'is', None, u'copyright', u'reserved', u'mit', u'is', u'license', None]
# 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14
assert tks_as_str(query_tokens_with_unknowns(qry)) == expected
result = idx.match(query_string=querys)
assert len(result) == 1
match = result[0]
assert match.qspan == Span(0, 4) | Span(6, 10)
assert match.ispan == Span(0, 9)
assert match.coverage() == 100
qtext, itext = get_texts(match)
assert qtext == 'copyright reserved mit is license [is] [the] copyright reserved mit is license'
assert itext == 'copyright reserved mit is license copyright reserved mit is license'
def test_QueryRun_does_not_end_with_None(self):
rule_text = 'Redistribution and use in source and binary forms, with or without modification, are permitted'
idx = index.LicenseIndex(
[Rule(stored_text=rule_text, license_expression='bsd')])
querys = '''
The
Redistribution and use in source and binary forms, with or without modification, are permitted.
Always
bar
modification
foo
'''
qry = Query(query_string=querys, idx=idx)
expected = [
None, 'redistribution', 'and', 'use', 'in', 'source', 'and',
'binary', 'forms', 'with', 'or', 'without', 'modification', 'are',
'permitted', None, None, 'modification', None
]
assert tks_as_str(qry.tokens, idx=idx) == [x for x in expected if x]
assert tks_as_str(query_tokens_with_unknowns(qry), idx=idx) == expected
assert len(qry.query_runs) == 2
qrun = qry.query_runs[0]
expected = [
'redistribution', 'and', 'use', 'in', 'source', 'and', 'binary',
'forms', 'with', 'or', 'without', 'modification', 'are',
'permitted'
]
assert tks_as_str(qrun.tokens, idx=idx) == expected
assert qrun.start == 0
assert qrun.end == 13
qrun = qry.query_runs[1]
expected = ['modification']
assert tks_as_str(qrun.tokens, idx=idx) == expected
assert qrun.start == 14
assert qrun.end == 14
def test_Query_known_and_unknown_positions(self):
rule_text = 'Redistribution and use in source and binary forms'
rule = Rule(stored_text=rule_text, license_expression='bsd')
legalese = set([
'redistribution',
'form',
])
idx = index.LicenseIndex([rule], _legalese=legalese)
querys = 'The new Redistribution and use in other form always'
qry = Query(query_string=querys, idx=idx, _test_mode=False)
# we have only 4 known positions in this query, hence only 4 entries there on a single line
# "Redistribution and use in"
assert qry.line_by_pos == [1, 1, 1, 1, 1]
# this show our 4 known token in this query with their known position
# "Redistribution and use in"
assert qry.tokens == [1, 2, 3, 4, 0]
# the first two tokens are unknown, then starting after "in" we have three trailing unknown.
assert qry.unknowns_by_pos == {3: 1, 4: 1, -1: 2}
# This shows how knowns and unknowns are blended
result = list(query_tokens_with_unknowns(qry))
expected = [
# The new
None,
None,
# Redistribution
1,
# and
2,
# use
3,
# in
4,
# other form always'
None,
0,
None
]
assert result == expected