def _update_query_with_dates(q: AdvancedQuery, date_data: MultiDict) \
-> AdvancedQuery:
filter_by = date_data['filter_by']
if filter_by == 'all_dates': # Nothing to do; all dates by default.
return q
elif filter_by == 'past_12':
one_year_ago = date.today() - relativedelta(months=12)
# Fix for these typing issues is coming soon!
# See: https://github.com/python/mypy/pull/4397
q.date_range = DateRange( # type: ignore
start_date=datetime(year=one_year_ago.year,
month=one_year_ago.month,
day=1,
hour=0,
minute=0,
second=0,
tzinfo=EASTERN))
elif filter_by == 'specific_year':
q.date_range = DateRange( # type: ignore
start_date=datetime(year=date_data['year'].year,
month=1,
day=1,
hour=0,
minute=0,
second=0,
tzinfo=EASTERN),
end_date=datetime(year=date_data['year'].year + 1,
month=1,
day=1,
hour=0,
minute=0,
second=0,
tzinfo=EASTERN),
)
elif filter_by == 'date_range':
if date_data['from_date']:
date_data['from_date'] = datetime.combine( # type: ignore
date_data['from_date'],
datetime.min.time(),
tzinfo=EASTERN)
if date_data['to_date']:
date_data['to_date'] = datetime.combine( # type: ignore
date_data['to_date'],
datetime.min.time(),
tzinfo=EASTERN)
q.date_range = DateRange( # type: ignore
start_date=date_data['from_date'],
end_date=date_data['to_date'],
)
if q.date_range:
q.date_range.date_type = date_data['date_type']
return q
def test_group_terms(self):
""":meth:`._group_terms` groups terms using logical precedence."""
query = AdvancedQuery(terms=FieldedSearchList([
FieldedSearchTerm(operator=None, field='title', term='muon'),
FieldedSearchTerm(operator='OR', field='title', term='gluon'),
FieldedSearchTerm(operator='NOT', field='title', term='foo'),
FieldedSearchTerm(operator='AND', field='title', term='boson'),
]))
expected = (
FieldedSearchTerm(operator=None, field='title', term='muon'),
'OR',
(
(
FieldedSearchTerm(operator='OR', field='title', term='gluon'),
'NOT',
FieldedSearchTerm(operator='NOT', field='title', term='foo')
),
'AND',
FieldedSearchTerm(operator='AND', field='title', term='boson')
)
)
try:
terms = advanced._group_terms(query)
except AssertionError:
self.fail('Should result in a single group')
self.assertEqual(expected, terms)
def _update_query_with_classification(q: AdvancedQuery, data: MultiDict) \
-> AdvancedQuery:
q.classification = ClassificationList()
archives = [('computer_science', 'cs'), ('economics', 'econ'),
('eess', 'eess'), ('mathematics', 'math'),
('q_biology', 'q-bio'), ('q_finance', 'q-fin'),
('statistics', 'stat')]
for field, archive in archives:
if data.get(field):
# Fix for these typing issues is coming soon!
# See: https://github.com/python/mypy/pull/4397
q.classification.append(
Classification(archive={'id': archive}) # type: ignore
)
if data.get('physics') and 'physics_archives' in data:
if 'all' in data['physics_archives']:
q.classification.append(
Classification(group={'id': 'grp_physics'}) # type: ignore
)
else:
q.classification.append(
Classification( # type: ignore
group={'id': 'grp_physics'},
archive={'id': data['physics_archives']}))
return q
def _update_query_with_terms(q: AdvancedQuery, terms_data: list) \
-> AdvancedQuery:
q.terms = FieldedSearchList([
FieldedSearchTerm(**term) # type: ignore
for term in terms_data if term['term']
])
return q
def _update_query_with_classification(q: AdvancedQuery,
data: MultiDict) -> AdvancedQuery:
q.classification = ClassificationList()
archives = [
("computer_science", "cs"),
("economics", "econ"),
("eess", "eess"),
("mathematics", "math"),
("q_biology", "q-bio"),
("q_finance", "q-fin"),
("statistics", "stat"),
]
for field, archive in archives:
if data.get(field):
# Fix for these typing issues is coming soon!
# See: https://github.com/python/mypy/pull/4397
q.classification.append(
Classification(archive={"id": archive}) # type: ignore
)
if data.get("physics") and "physics_archives" in data:
if "all" in data["physics_archives"]:
q.classification.append(
Classification(group={"id": "grp_physics"}) # type: ignore
)
else:
q.classification.append(
Classification( # type: ignore
group={"id": "grp_physics"},
archive={"id": data["physics_archives"]},
))
return q
def test_group_terms(self):
""":meth:`._group_terms` groups terms using logical precedence."""
query = AdvancedQuery(terms=FieldedSearchList([
FieldedSearchTerm(operator=None, field="title", term="muon"),
FieldedSearchTerm(operator="OR", field="title", term="gluon"),
FieldedSearchTerm(operator="NOT", field="title", term="foo"),
FieldedSearchTerm(operator="AND", field="title", term="boson"),
]))
expected = (
FieldedSearchTerm(operator=None, field="title", term="muon"),
"OR",
(
(
FieldedSearchTerm(operator="OR",
field="title",
term="gluon"),
"NOT",
FieldedSearchTerm(operator="NOT",
field="title",
term="foo"),
),
"AND",
FieldedSearchTerm(operator="AND", field="title", term="boson"),
),
)
try:
terms = advanced._group_terms(query)
except AssertionError:
self.fail("Should result in a single group")
self.assertEqual(expected, terms)
def test_specific_year_is_selected(self):
"""Start and end dates are set, one year apart."""
date_data = {
'filter_by': 'specific_year',
'year': date(year=1999, month=1, day=1)
}
q = advanced._update_query_with_dates(AdvancedQuery(), date_data)
self.assertIsInstance(q, AdvancedQuery)
self.assertEqual(q.date_range.end_date.date(),
date(year=2000, month=1, day=1))
self.assertEqual(q.date_range.start_date.date(),
date(year=1999, month=1, day=1))
def _query_from_form(form: forms.AdvancedSearchForm) -> AdvancedQuery:
"""
Generate a :class:`.AdvancedQuery` from valid :class:`.AdvancedSearchForm`.
Parameters
----------
form : :class:`.AdvancedSearchForm`
Presumed to be filled and valid.
Returns
-------
:class:`.AdvancedQuery`
"""
q = AdvancedQuery()
q = _update_query_with_dates(q, form.date.data)
q = _update_query_with_terms(q, form.terms.data)
q = _update_query_with_classification(q, form.classification.data)
q.include_cross_list = form.classification.include_cross_list.data \
== form.classification.INCLUDE_CROSS_LIST
if form.include_older_versions.data:
q.include_older_versions = True
order = form.order.data
if order and order != 'None':
q.order = order
q.hide_abstracts = form.abstracts.data == form.HIDE_ABSTRACTS
return q
def test_specific_year_is_selected(self):
"""Start and end dates are set, one year apart."""
date_data = {
"filter_by": "specific_year",
"year": date(year=1999, month=1, day=1),
"date_type": "submitted_date",
}
q = advanced._update_query_with_dates(AdvancedQuery(), date_data)
self.assertIsInstance(q, AdvancedQuery)
self.assertEqual(q.date_range.end_date.date(),
date(year=2000, month=1, day=1))
self.assertEqual(q.date_range.start_date.date(),
date(year=1999, month=1, day=1))
def test_date_range_is_selected(self):
"""Start and end dates are set based on selection."""
from_date = date(year=1999, month=7, day=3)
to_date = date(year=1999, month=8, day=5)
date_data = {
'filter_by': 'date_range',
'from_date': from_date,
'to_date': to_date,
}
q = advanced._update_query_with_dates(AdvancedQuery(), date_data)
self.assertIsInstance(q, AdvancedQuery)
self.assertEqual(q.date_range.end_date.date(), to_date)
self.assertEqual(q.date_range.start_date.date(), from_date)
def test_date_range_is_selected(self):
"""Start and end dates are set based on selection."""
from_date = date(year=1999, month=7, day=3)
to_date = date(year=1999, month=8, day=5)
date_data = {
"filter_by": "date_range",
"from_date": from_date,
"to_date": to_date,
"date_type": "submitted_date",
}
q = advanced._update_query_with_dates(AdvancedQuery(), date_data)
self.assertIsInstance(q, AdvancedQuery)
self.assertEqual(q.date_range.end_date.date(), to_date)
self.assertEqual(q.date_range.start_date.date(), from_date)
def test_advanced_query(self, mock_Elasticsearch, mock_Search):
""":class:`.index.search` supports :class:`AdvancedQuery`."""
mock_results = mock.MagicMock()
mock_results.__getitem__.return_value = {'total': 53}
rdata = dict(authors=[{'full_name': 'N. Ame'}],
owners=[{'full_name': 'N. Ame'}],
submitter={'full_name': 'N. Ame'},
paper_id='1234.56789')
mock_result = mock.MagicMock(_d_=rdata, **rdata)
mock_result.meta.score = 1
mock_results.__iter__.return_value = [mock_result]
mock_Search.execute.return_value = mock_results
# Support the chaining API for py-ES.
mock_Search.return_value = mock_Search
mock_Search.filter.return_value = mock_Search
mock_Search.highlight.return_value = mock_Search
mock_Search.highlight_options.return_value = mock_Search
mock_Search.query.return_value = mock_Search
mock_Search.sort.return_value = mock_Search
mock_Search.__getitem__.return_value = mock_Search
query = AdvancedQuery(
order='relevance',
size=10,
date_range=DateRange(
start_date=datetime.now() - timedelta(days=5),
end_date=datetime.now()
),
classification=ClassificationList([
Classification(
group={'id': 'physics'},
archive={'id': 'physics'},
category={'id': 'hep-th'}
)
]),
terms=FieldedSearchList([
FieldedSearchTerm(operator='AND', field='title', term='foo'),
FieldedSearchTerm(operator='AND', field='author', term='joe'),
FieldedSearchTerm(operator='OR', field='abstract', term='hmm'),
FieldedSearchTerm(operator='NOT', field='comments', term='eh'),
FieldedSearchTerm(operator='AND', field='journal_ref',
term='jref (1999) 1:2-3'),
FieldedSearchTerm(operator='AND', field='acm_class',
term='abc123'),
FieldedSearchTerm(operator='AND', field='msc_class',
term='abc123'),
FieldedSearchTerm(operator='OR', field='report_num',
term='abc123'),
FieldedSearchTerm(operator='OR', field='doi',
term='10.01234/56789'),
FieldedSearchTerm(operator='OR', field='orcid',
term='0000-0000-0000-0000'),
FieldedSearchTerm(operator='OR', field='author_id',
term='Bloggs_J'),
])
)
document_set = index.SearchSession.search(query)
# self.assertIsInstance(document_set, DocumentSet)
self.assertEqual(document_set['metadata']['start'], 0)
self.assertEqual(document_set['metadata']['total'], 53)
self.assertEqual(document_set['metadata']['current_page'], 1)
self.assertEqual(document_set['metadata']['total_pages'], 6)
self.assertEqual(document_set['metadata']['size'], 10)
self.assertEqual(len(document_set['results']), 1)
def test_all_dates_is_selected(self):
"""Query does not select on date."""
date_data = {"filter_by": "all_dates", "date_type": "submitted_date"}
q = advanced._update_query_with_dates(AdvancedQuery(), date_data)
self.assertIsInstance(q, AdvancedQuery)
self.assertIsNone(q.date_range)
def test_advanced_query(self, mock_Elasticsearch, mock_Search):
""":class:`.index.search` supports :class:`AdvancedQuery`."""
mock_results = mock.MagicMock()
mock_results.__getitem__.return_value = {"total": 53}
rdata = mock_rdata()
mock_result = mock.MagicMock(_d_=rdata, **rdata)
mock_result.meta.score = 1
mock_results.__iter__.return_value = [mock_result]
mock_Search.execute.return_value = mock_results
# Support the chaining API for py-ES.
mock_Search.return_value = mock_Search
mock_Search.filter.return_value = mock_Search
mock_Search.highlight.return_value = mock_Search
mock_Search.highlight_options.return_value = mock_Search
mock_Search.query.return_value = mock_Search
mock_Search.sort.return_value = mock_Search
mock_Search.__getitem__.return_value = mock_Search
query = AdvancedQuery(
order="relevance",
size=10,
date_range=DateRange(
start_date=datetime.now() - timedelta(days=5),
end_date=datetime.now(),
),
classification=ClassificationList([
Classification(
group={"id": "physics"},
archive={"id": "physics"},
category={"id": "hep-th"},
)
]),
terms=FieldedSearchList([
FieldedSearchTerm(operator="AND", field="title", term="foo"),
FieldedSearchTerm(operator="AND", field="author", term="joe"),
FieldedSearchTerm(operator="OR", field="abstract", term="hmm"),
FieldedSearchTerm(operator="NOT", field="comments", term="eh"),
FieldedSearchTerm(
operator="AND",
field="journal_ref",
term="jref (1999) 1:2-3",
),
FieldedSearchTerm(operator="AND",
field="acm_class",
term="abc123"),
FieldedSearchTerm(operator="AND",
field="msc_class",
term="abc123"),
FieldedSearchTerm(operator="OR",
field="report_num",
term="abc123"),
FieldedSearchTerm(operator="OR",
field="doi",
term="10.01234/56789"),
FieldedSearchTerm(
operator="OR",
field="orcid",
term="0000-0000-0000-0000",
),
FieldedSearchTerm(operator="OR",
field="author_id",
term="Bloggs_J"),
]),
)
document_set = index.SearchSession.search(query)
# self.assertIsInstance(document_set, DocumentSet)
self.assertEqual(document_set["metadata"]["start"], 0)
self.assertEqual(document_set["metadata"]["total_results"], 53)
self.assertEqual(document_set["metadata"]["current_page"], 1)
self.assertEqual(document_set["metadata"]["total_pages"], 6)
self.assertEqual(document_set["metadata"]["size"], 10)
self.assertEqual(len(document_set["results"]), 1)
def _update_query_with_dates(q: AdvancedQuery,
date_data: MultiDict) -> AdvancedQuery:
filter_by = date_data["filter_by"]
if filter_by == "all_dates": # Nothing to do; all dates by default.
return q
elif filter_by == "past_12":
one_year_ago = date.today() - relativedelta(months=12)
# Fix for these typing issues is coming soon!
# See: https://github.com/python/mypy/pull/4397
q.date_range = DateRange( # type: ignore
start_date=datetime(
year=one_year_ago.year,
month=one_year_ago.month,
day=1,
hour=0,
minute=0,
second=0,
tzinfo=consts.EASTERN,
))
elif filter_by == "specific_year":
q.date_range = DateRange( # type: ignore
start_date=datetime(
year=date_data["year"].year,
month=1,
day=1,
hour=0,
minute=0,
second=0,
tzinfo=consts.EASTERN,
),
end_date=datetime(
year=date_data["year"].year + 1,
month=1,
day=1,
hour=0,
minute=0,
second=0,
tzinfo=consts.EASTERN,
),
)
elif filter_by == "date_range":
if date_data["from_date"]:
date_data["from_date"] = datetime.combine( # type: ignore
date_data["from_date"],
datetime.min.time(),
tzinfo=consts.EASTERN,
)
if date_data["to_date"]:
date_data["to_date"] = datetime.combine( # type: ignore
date_data["to_date"],
datetime.min.time(),
tzinfo=consts.EASTERN,
)
q.date_range = DateRange( # type: ignore
start_date=date_data["from_date"],
end_date=date_data["to_date"])
if q.date_range:
q.date_range.date_type = date_data["date_type"]
return q
def test_all_dates_is_selected(self):
"""Query does not select on date."""
date_data = {'filter_by': 'all_dates'}
q = advanced._update_query_with_dates(AdvancedQuery(), date_data)
self.assertIsInstance(q, AdvancedQuery)
self.assertIsNone(q.date_range)