def test_dates(self):
testcase = {
"created:2017-05-02": str(SQ(created__gte='2017-05-02T00:00:00Z') &
SQ(created__lt='2017-05-03T00:00:00Z')),
"created:2017-05": str(SQ(created__gte='2017-05-01T00:00:00Z') &
SQ(created__lt='2017-05-02T00:00:00Z')),
"created:2017": str(SQ(created__gte='2017-01-01T00:00:00Z') &
SQ(created__lt='2017-01-02T00:00:00Z')),
"modified:2017-05-02": str(SQ(modified__gte='2017-05-02T00:00:00Z') &
SQ(modified__lt='2017-05-03T00:00:00Z')),
"modified:2017-05": str(SQ(modified__gte='2017-05-01T00:00:00Z') &
SQ(modified__lt='2017-05-02T00:00:00Z')),
"modified:2017": str(SQ(modified__gte='2017-01-01T00:00:00Z') &
SQ(modified__lt='2017-01-02T00:00:00Z')),
"start_date:2017-05-02": str(SQ(start_date__gte='2017-05-02T00:00:00Z') &
SQ(start_date__lt='2017-05-03T00:00:00Z')),
"start_date:2017-05": str(SQ(start_date__gte='2017-05-01T00:00:00Z') &
SQ(start_date__lt='2017-05-02T00:00:00Z')),
"start_date:2017": str(SQ(start_date__gte='2017-01-01T00:00:00Z') &
SQ(start_date__lt='2017-01-02T00:00:00Z')),
"end_date:2017-05-02": str(SQ(end_date__gte='2017-05-02T00:00:00Z') &
SQ(end_date__lt='2017-05-03T00:00:00Z')),
"end_date:2017-05": str(SQ(end_date__gte='2017-05-01T00:00:00Z') &
SQ(end_date__lt='2017-05-02T00:00:00Z')),
"end_date:2017": str(SQ(end_date__gte='2017-01-01T00:00:00Z') &
SQ(end_date__lt='2017-01-02T00:00:00Z')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_basic_parse(self):
testcase = {
"note": str(SQ(content="note")),
'"need note"': str(SQ(content__exact="need note")),
"author:admin": str(SQ(creator="admin")),
"first_author:admin": str(SQ(author="admin")),
"author:admin notes": str(SQ(creator="admin") & SQ(content="notes")),
"author:admin OR notes": str(SQ(creator="admin") | SQ(content="notes")),
'title:"need note"': str(SQ(title__exact="need note")),
# "need note ?????": str(SQ(content="need") & SQ(content="note") &
# SQ(content=u"?") & SQ(content=u"?") & SQ(content=u"?") &
# SQ(content=u"?") & SQ(content=u"?")),
"need note NOT used": str(SQ(content="need") &
SQ(content="note") & ~ SQ(content="used")),
"(a AND b) OR (c AND d)": str((SQ(content="a") & SQ(content="b")) |
(SQ(content="c") & SQ(content="d"))),
"a AND b OR (c AND d)": str(SQ(content="a") & SQ(content="b") |
(SQ(content="c") & SQ(content="d"))),
'"a AND b" OR "(c AND d)"': str(SQ(content__exact="a AND b") |
SQ(content__exact="(c AND d)")),
'"notes done" OR papaya': str(SQ(content__exact="notes done") |
SQ(content="papaya")),
'"a AND b" OR (c AND d)': str(SQ(content__exact="a AND b") |
(SQ(content="c") & SQ(content="d"))),
'subject:"exp>20"': str(SQ(subject__exact="exp>20")),
'subject:"HP employee"': str(SQ(subject__exact="HP employee")),
'subject:"HP employee" OR something': str(SQ(subject__exact="HP employee") |
SQ(content='something')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_exceptions(self):
testcase = {
"foo:bar": FieldNotRecognizedError,
"created:20170": MalformedDateError,
"created:2017-30": MalformedDateError,
"created:2017-12-64": MalformedDateError,
"abstract>foo": InequalityNotAllowedError,
"(abstract:something": MatchingBracketsNotFoundError
}
parser = ParseSQ()
for case in testcase.keys():
with self.assertRaises(testcase[case]):
parser.parse(case)
def test_operators(self):
testcase = {
"note": str(SQ(content="note")),
"need -note": str(SQ(content="need") & ~SQ(content="note")),
"need +note": str(SQ(content="need") & SQ(content="note")),
"need+note": str(SQ(content="need+note")),
"iphone AND NOT subject:10": str(SQ(content="iphone") & ~SQ(
subject="10")),
"NOT subject:10": str(~SQ(subject="10")),
"subject:10": str(SQ(subject="10")),
"-subject:10": str(~SQ(subject="10")),
"subject:-10": str(SQ(subject="-10")),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_operators(self):
testcase = {
# removed '+', '-' syntaxes 4/5/2019.
# test cases modified accordingly.
'note':
str(SQ(content='note')),
'"note"':
str(SQ(content__exact='note')),
'need -note':
str(SQ(content='need') & SQ(content='-note')),
'"need -note"':
str(SQ(content__exact='need -note')),
'need +note':
str(SQ(content='need') & SQ(content='+note')),
'need+note':
str(SQ(content='need+note')),
'iphone AND NOT subject:10':
str(SQ(content='iphone') & ~SQ(subject='10')),
'iphone OR NOT subject:10':
str(SQ(content='iphone') | ~SQ(subject='10')),
'NOT subject:10':
str(~SQ(subject='10')),
'subject:10':
str(SQ(subject='10')),
'-subject:10':
str(SQ(content='-subject:10')),
'subject:-10':
str(SQ(subject='-10')),
# all keywords accepted, non-matches are treated literally
'foo:bar':
str(SQ(content='foo:bar')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_parse_with_new_default(self):
testcase = {
'helo again bye': {'sq': str(SQ(content='helo') | SQ(content='again') |
SQ(content='bye')),
'operator': 'OR'},
'helo again AND bye': {
'sq': str((SQ(content='helo') | SQ(content='again')) & SQ(content='bye')),
'operator': 'OR'},
'helo again AND bye run': {
'sq': str(((SQ(content='helo') | SQ(content='again')) &
SQ(content='bye')) | SQ(content='run')),
'operator': 'OR'},
}
for case in testcase.keys():
parser = ParseSQ(testcase[case]['operator'])
self.assertEqual(str(parser.parse(case, )), testcase[case]['sq'])
def test_inequalities(self):
testcase = {
"north<=50.0": str(SQ(north__lte='50.0')),
"north<50.0": str(SQ(north__lt='50.0')),
"north>=50.0": str(SQ(north__gte='50.0')),
"north>50.0": str(SQ(north__gt='50.0')),
"east<=50.0": str(SQ(east__lte='50.0')),
"east<50.0": str(SQ(east__lt='50.0')),
"east>=50.0": str(SQ(east__gte='50.0')),
"east>50.0": str(SQ(east__gt='50.0')),
"created>2017-05-02": str(SQ(created__gte='2017-05-03T00:00:00Z')),
"created>=2017-05-02": str(SQ(created__gte='2017-05-02T00:00:00Z')),
"created<2017-05-02": str(SQ(created__lt='2017-05-02T00:00:00Z')),
"created<=2017-05-02": str(SQ(created__lt='2017-05-03T00:00:00Z')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def handle(self, *args, **options):
if len(options['queries']
) > 0: # an array of resource short_id to check.
query = ' '.join(options['queries'])
sqs = SearchQuerySet().all()
parser = ParseSQ()
parsed = parser.parse(query)
sqs = sqs.filter(parsed)
print("QUERY '{}' PARSED {}".format(query, str(parsed)))
for result in list(sqs):
stored = result.get_stored_fields()
print((" {}: {} {} {} {}".format(str(stored['short_id']),
str(stored['title']),
str(stored['author']),
str(stored['created']),
str(stored['modified']))))
else:
print("no queries to try")
def handle(self, *args, **options):
if len(options['queries']) > 0: # an array of resource short_id to check.
query = ' '.join(options['queries'])
sqs = SearchQuerySet().all()
parser = ParseSQ()
parsed = parser.parse(query)
sqs = sqs.filter(parsed)
print("QUERY '{}' PARSED {}".format(query, str(parsed)))
for result in list(sqs):
stored = result.get_stored_fields()
print(" {}: {} {} {} {}".format(
unicode(stored['short_id']).encode('ascii', 'replace'),
unicode(stored['title']).encode('ascii', 'replace'),
unicode(stored['author']).encode('ascii', 'replace'),
unicode(stored['created']).encode('ascii', 'replace'),
unicode(stored['modified']).encode('ascii', 'replace')))
else:
print("no queries to try")
def test_dates(self):
testcase = {
"created:2017-05-02":
str(
SQ(created__gte='2017-05-02T00:00:00Z')
& SQ(created__lt='2017-05-03T00:00:00Z')),
"created:2017-05":
str(
SQ(created__gte='2017-05-01T00:00:00Z')
& SQ(created__lt='2017-05-02T00:00:00Z')),
"created:2017":
str(
SQ(created__gte='2017-01-01T00:00:00Z')
& SQ(created__lt='2017-01-02T00:00:00Z')),
"modified:2017-05-02":
str(
SQ(modified__gte='2017-05-02T00:00:00Z')
& SQ(modified__lt='2017-05-03T00:00:00Z')),
"modified:2017-05":
str(
SQ(modified__gte='2017-05-01T00:00:00Z')
& SQ(modified__lt='2017-05-02T00:00:00Z')),
"modified:2017":
str(
SQ(modified__gte='2017-01-01T00:00:00Z')
& SQ(modified__lt='2017-01-02T00:00:00Z')),
"start_date:2017-05-02":
str(
SQ(start_date__gte='2017-05-02T00:00:00Z')
& SQ(start_date__lt='2017-05-03T00:00:00Z')),
"start_date:2017-05":
str(
SQ(start_date__gte='2017-05-01T00:00:00Z')
& SQ(start_date__lt='2017-05-02T00:00:00Z')),
"start_date:2017":
str(
SQ(start_date__gte='2017-01-01T00:00:00Z')
& SQ(start_date__lt='2017-01-02T00:00:00Z')),
"end_date:2017-05-02":
str(
SQ(end_date__gte='2017-05-02T00:00:00Z')
& SQ(end_date__lt='2017-05-03T00:00:00Z')),
"end_date:2017-05":
str(
SQ(end_date__gte='2017-05-01T00:00:00Z')
& SQ(end_date__lt='2017-05-02T00:00:00Z')),
"end_date:2017":
str(
SQ(end_date__gte='2017-01-01T00:00:00Z')
& SQ(end_date__lt='2017-01-02T00:00:00Z')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_exceptions(self):
testcase = {
# This exception removed 4/5/2019: "foo:bar": FieldNotRecognizedError,
"created:20170": MalformedDateError,
"created:2017-30": MalformedDateError,
"created:2017-12-64": MalformedDateError,
"abstract>foo": InequalityNotAllowedError,
"(abstract:something": MatchingBracketsNotFoundError
}
parser = ParseSQ()
for case in testcase.keys():
with self.assertRaises(testcase[case]):
parser.parse(case)
def test_basic_parse(self):
testcase = {
"note":
str(SQ(content="note")),
'"need note"':
str(SQ(content__exact="need note")),
"author:admin":
str(SQ(creator="admin")),
"first_author:admin":
str(SQ(author="admin")),
"author:admin notes":
str(SQ(creator="admin") & SQ(content="notes")),
"author:admin OR notes":
str(SQ(creator="admin") | SQ(content="notes")),
'title:"need note"':
str(SQ(title__exact="need note")),
# "need note ?????": str(SQ(content="need") & SQ(content="note") &
# SQ(content=u"?") & SQ(content=u"?") & SQ(content=u"?") &
# SQ(content=u"?") & SQ(content=u"?")),
"need note NOT used":
str(SQ(content="need") & SQ(content="note") & ~SQ(content="used")),
"(a AND b) OR (c AND d)":
str((SQ(content="a") & SQ(content="b"))
| (SQ(content="c") & SQ(content="d"))),
"a AND b OR (c AND d)":
str(
SQ(content="a") & SQ(content="b")
| (SQ(content="c") & SQ(content="d"))),
'"a AND b" OR "(c AND d)"':
str(SQ(content__exact="a AND b") | SQ(content__exact="(c AND d)")),
'"notes done" OR papaya':
str(SQ(content__exact="notes done") | SQ(content="papaya")),
'"a AND b" OR (c AND d)':
str(
SQ(content__exact="a AND b")
| (SQ(content="c") & SQ(content="d"))),
'subject:"exp>20"':
str(SQ(subject__exact="exp>20")),
'subject:"HP employee"':
str(SQ(subject__exact="HP employee")),
'subject:"HP employee" OR something':
str(SQ(subject__exact="HP employee") | SQ(content='something')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_inequalities(self):
testcase = {
"north<=50.0": str(SQ(north__lte='50.0')),
"north<50.0": str(SQ(north__lt='50.0')),
"north>=50.0": str(SQ(north__gte='50.0')),
"north>50.0": str(SQ(north__gt='50.0')),
"east<=50.0": str(SQ(east__lte='50.0')),
"east<50.0": str(SQ(east__lt='50.0')),
"east>=50.0": str(SQ(east__gte='50.0')),
"east>50.0": str(SQ(east__gt='50.0')),
"created>2017-05-02": str(SQ(created__gte='2017-05-03T00:00:00Z')),
"created>=2017-05-02":
str(SQ(created__gte='2017-05-02T00:00:00Z')),
"created<2017-05-02": str(SQ(created__lt='2017-05-02T00:00:00Z')),
"created<=2017-05-02": str(SQ(created__lt='2017-05-03T00:00:00Z')),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def test_operators(self):
testcase = {
"note":
str(SQ(content="note")),
"need -note":
str(SQ(content="need") & ~SQ(content="note")),
"need +note":
str(SQ(content="need") & SQ(content="note")),
"need+note":
str(SQ(content="need+note")),
"iphone AND NOT subject:10":
str(SQ(content="iphone") & ~SQ(subject="10")),
"NOT subject:10":
str(~SQ(subject="10")),
"subject:10":
str(SQ(subject="10")),
"-subject:10":
str(~SQ(subject="10")),
"subject:-10":
str(SQ(subject="-10")),
}
parser = ParseSQ()
for case in testcase.keys():
self.assertEqual(str(parser.parse(case)), testcase[case])
def search(self):
self.parse_error = None # error return from parser
sqs = self.searchqueryset.all().filter(replaced=False)
if self.cleaned_data.get('q'):
# The prior code corrected for an failed match of complete words, as documented
# in issue #2308. This version instead uses an advanced query syntax in which
# "word" indicates an exact match and the bare word indicates a stemmed match.
cdata = self.cleaned_data.get('q')
try:
parser = ParseSQ()
parsed = parser.parse(cdata)
sqs = sqs.filter(parsed)
except ValueError as e:
sqs = self.searchqueryset.none()
self.parse_error = "Value error: {}. No matches. Please try again".format(e.value)
return sqs
except MatchingBracketsNotFoundError as e:
sqs = self.searchqueryset.none()
self.parse_error = "{} No matches. Please try again.".format(e.value)
return sqs
except MalformedDateError as e:
sqs = self.searchqueryset.none()
self.parse_error = "{} No matches. Please try again.".format(e.value)
return sqs
except FieldNotRecognizedError as e:
sqs = self.searchqueryset.none()
self.parse_error = \
("{} Field delimiters include title, contributor, subject, etc. " +
"Please try again.")\
.format(e.value)
return sqs
except InequalityNotAllowedError as e:
sqs = self.searchqueryset.none()
self.parse_error = "{} No matches. Please try again.".format(e.value)
return sqs
geo_sq = None
if self.cleaned_data['NElng'] and self.cleaned_data['SWlng']:
if float(self.cleaned_data['NElng']) > float(self.cleaned_data['SWlng']):
geo_sq = SQ(east__lte=float(self.cleaned_data['NElng']))
geo_sq.add(SQ(east__gte=float(self.cleaned_data['SWlng'])), SQ.AND)
else:
geo_sq = SQ(east__gte=float(self.cleaned_data['SWlng']))
geo_sq.add(SQ(east__lte=float(180)), SQ.OR)
geo_sq.add(SQ(east__lte=float(self.cleaned_data['NElng'])), SQ.AND)
geo_sq.add(SQ(east__gte=float(-180)), SQ.AND)
if self.cleaned_data['NElat'] and self.cleaned_data['SWlat']:
# latitude might be specified without longitude
if geo_sq is None:
geo_sq = SQ(north__lte=float(self.cleaned_data['NElat']))
else:
geo_sq.add(SQ(north__lte=float(self.cleaned_data['NElat'])), SQ.AND)
geo_sq.add(SQ(north__gte=float(self.cleaned_data['SWlat'])), SQ.AND)
if geo_sq is not None:
sqs = sqs.filter(geo_sq)
# Check to see if a start_date was chosen.
start_date = self.cleaned_data['start_date']
end_date = self.cleaned_data['end_date']
# allow overlapping ranges
# cs < s < ce OR s < cs => s < ce
# AND
# cs < e < ce OR e > ce => cs < e
if start_date and end_date:
sqs = sqs.filter(SQ(end_date__gte=start_date) &
SQ(start_date__lte=end_date))
elif start_date:
sqs = sqs.filter(SQ(end_date__gte=start_date))
elif end_date:
sqs = sqs.filter(SQ(start_date__lte=end_date))
if self.cleaned_data['coverage_type']:
sqs = sqs.filter(coverage_types__in=[self.cleaned_data['coverage_type']])
creator_sq = None
contributor_sq = None
owner_sq = None
subject_sq = None
resource_type_sq = None
availability_sq = None
# We need to process each facet to ensure that the field name and the
# value are quoted correctly and separately:
for facet in self.selected_facets:
if ":" not in facet:
continue
field, value = facet.split(":", 1)
value = sqs.query.clean(value)
if value:
if "creator" in field:
if creator_sq is None:
creator_sq = SQ(creator__exact=value)
else:
creator_sq.add(SQ(creator__exact=value), SQ.OR)
if "contributor" in field:
if contributor_sq is None:
contributor_sq = SQ(contributor__exact=value)
else:
contributor_sq.add(SQ(contributor__exact=value), SQ.OR)
elif "owner" in field:
if owner_sq is None:
owner_sq = SQ(owner__exact=value)
else:
owner_sq.add(SQ(owner__exact=value), SQ.OR)
elif "subject" in field:
if subject_sq is None:
subject_sq = SQ(subject__exact=value)
else:
subject_sq.add(SQ(subject__exact=value), SQ.OR)
elif "resource_type" in field:
if resource_type_sq is None:
resource_type_sq = SQ(resource_type__exact=value)
else:
resource_type_sq.add(SQ(resource_type__exact=value), SQ.OR)
elif "availability" in field:
if availability_sq is None:
availability_sq = SQ(availability__exact=value)
else:
availability_sq.add(SQ(availability__exact=value), SQ.OR)
else:
continue
if creator_sq is not None:
sqs = sqs.filter(creator_sq)
if contributor_sq is not None:
sqs = sqs.filter(contributor_sq)
if owner_sq is not None:
sqs = sqs.filter(owner_sq)
if subject_sq is not None:
sqs = sqs.filter(subject_sq)
if resource_type_sq is not None:
sqs = sqs.filter(resource_type_sq)
if availability_sq is not None:
sqs = sqs.filter(availability_sq)
return sqs
