def test_iter(self):
long_results = Results({
'response': {
'docs': [{
'id': 1
}, {
'id': 2
}, {
'id': 3
}],
'numFound': 7,
},
})
to_iter = list(long_results)
self.assertEqual(to_iter[0], {'id': 1})
self.assertEqual(to_iter[1], {'id': 2})
self.assertEqual(to_iter[2], {'id': 3})
def test_iter(self):
long_results = Results({
"response": {
"docs": [{
"id": 1
}, {
"id": 2
}, {
"id": 3
}],
"numFound": 7
}
})
to_iter = list(long_results)
self.assertEqual(to_iter[0], {"id": 1})
self.assertEqual(to_iter[1], {"id": 2})
self.assertEqual(to_iter[2], {"id": 3})
def test_update_index_doc(self):
cls = self._get_mixin_class()
cls.index.search.return_value = Results({
'response': {
'docs': [{
'khe_id': 'aaa',
'title': 'Test Model',
'name': 'test-model',
'khe_description': 'Test Description',
'site_id': 'test',
'entity_type': 'test_doc',
'index_id': '1',
}],
'numFound': 1,
}
})
def _add(doctype, doc):
assert_equals('test_doc', doctype)
assert_true(doc)
assert_true(doc.get('khe_id'))
assert_true(doc.get('title'))
assert_true(doc.get('name'))
assert_true(doc.get('khe_description'))
assert_true(doc.get('site_id'))
assert_true(doc.get('index_id'))
assert_equals('test_doc', doc.get('entity_type'))
cls.index.add.side_effect = _add
cls.update_index_doc({
'id': 'aaa',
'title': 'Test Model',
'name': 'test-model',
'description': 'Test Description',
})
cls.index.search.assert_called_once_with('test_doc', q='khe_id:aaa')
cls.index.remove.assert_called_once_with('test_doc', khe_id='aaa')
cls.index.add.assert_called_once()
def test_search_all_args(self):
solr_conn_mock = Mock()
index = Index()
index.get_connection = Mock()
index.get_connection.return_value = solr_conn_mock
solr_conn_mock.search.return_value = Results({
'response': {
'docs': [{
'id': 'aaa',
'p1': 'v1',
'p2': 'v2',
}],
'numFound': 1,
}
})
query = {
'q': 'id:aaa',
'fq': {
'p1': 'v1',
'p2': 'v2',
},
'rows': 10,
'start': 4,
'sort': 'qsort'
}
results = index.search('test_doc', **query)
solr_conn_mock.search.assert_called_once_with(**{
'q': 'id:aaa',
'fq': ['p1:v1', 'p2:v2', 'entity_type:test_doc'],
'rows': 10,
'start': 4,
'sort': 'qsort'
})
assert_true(results is not None)
assert_equals(len(results.docs), 1, 'Expected at least 1 result')
def test_infer_transactional(self):
user_intents = MagicMock()
nlp = MagicMock()
research_question = MagicMock()
model_locator = MagicMock()
rq = {
'id': 'question-id',
'theme_id': 'theme-id',
'theme_title': 'Theme',
'sub_theme_id': 'sub-theme-id',
'sub_theme_title': 'SubTheme',
'title': 'RQ Title',
}
research_question.search_index.return_value = Results(
{'response': {
'docs': [rq],
'numFound': 1,
}})
extractor = UserIntentsExtractor(user_intents, nlp, research_question,
model_locator)
context = {}
user_intent = UserIntents()
query = UserQuery(id='query-id',
user_id='user-1',
query_type='package_search',
query_text='Some text')
result = extractor.infer_transactional(context, user_intent, query)
assert_true(result is not None)
assert_equals(result.get('research_question'), rq)
assert_equals(result.get('theme'), 'theme-id')
assert_equals(result.get('theme_value'), 'Theme')
assert_equals(result.get('sub_theme'), 'sub-theme-id')
assert_equals(result.get('sub_theme_value'), 'SubTheme')
assert_equals(result.get('predicted_values'), False)
def test_len(self):
small_results = Results([{'id': 1}, {'id': 2}], 2)
self.assertEqual(len(small_results), 2)
wrong_hits_results = Results([{'id': 1}, {'id': 2}, {'id': 3}], 7)
self.assertEqual(len(wrong_hits_results), 3)
def standard(name=None):
# do a match all request
query = "*:*"
start = 0
user = None
sort_criteria = None
if request.method == 'POST' and 'search_box' in request.form:
query = request.form['search_box']
else:
if request.args.get('q'):
query = request.args.get('q')
if request.method == 'POST' and "user" in request.form:
user = request.form['user']
else:
if request.args.get('user'):
user = request.args.get('user')
if request.args.get('start'):
start = request.args.get('start')
fq = []
if request.args.get('fq'):
fq = request.args.getlist('fq')
active = "Results"
if request.args.get('active'):
active = request.args.get('active')
if request.args.get('sort_criteria'):
sort_criteria = request.args.get('sort_criteria')
dsn_results = "data_source_name:HistoricalPrices"
source_filters = []
group = "false"
group_field = "symbol"
if active == "Results":
source_filters.append("-" + dsn_results)
else: #Historical, do grouping
source_filters.append(dsn_results)
group = "true"
if sort_criteria == None:
sort_criteria = "trade_date"
# &facet.date=timestamp&facet.date.start=2013-10-08T14:17:49.04Z&facet.date.end=NOW/DAY%2B1DAY&facet.date.gap=%2B1HOUR
app.logger.info("Query: " + query)
kwargs = {
"qt": "/lucid",
"facet": "true",
"start": start,
"fl": "*,score",
"facet.date": "timestamp",
"facet.date.start": "NOW/DAY-30DAY",
"facet.date.end": "NOW/DAY+1DAY",
"facet.date.gap": "+1DAY",
"facet.date.other": "all",
"facet.range": ["open", "close", "volume"],
"facet.range.start": "0",
"facet.range.end": "1000",
"facet.range.gap": "100",
"facet.range.other": "all",
"facet.mincount": "1",
"f.open.facet.limit": "5",
"f.close.facet.limit": "5",
"f.close.open.limit": "5",
"f.volume.facet.limit": "5",
"f.volume.facet.range.gap": "500000",
"f.volume.facet.range.start": "10000",
"f.volume.facet.range.end": "5000000",
"facet.pivot":
["open,close,volume", "attr_retweetcount,attr_username"],
"stats": "true",
"stats.field": ["open", "close", "volume"],
"fq": source_filters
}
if fq:
kwargs['fq'] = fq
#the_role = "DEFAULT"
if sort_criteria:
kwargs['group.sort'] = sort_criteria + " desc"
if active == "Historical":
kwargs['group'] = group
kwargs['group.field'] = group_field
kwargs['group.limit'] = 30
if user and user != 'none':
kwargs['user'] = user
# we have a user, let's see what roles they play
#/api/collections/collection/roles/role
print "User: "******"/roles", method="GET")
#{u'groups': [], u'users': [u'admin'], u'filters': [u'*:*'], u'name': u'DEFAULT'}
#{u'groups': [], u'users': [u'user.10'], u'filters': [u'symbol:AES'], u'name': u'user10'}
for role in roles:
#print user + ", " + role['users'][0]
for role_user in role['users']:
if user == role_user:
#print role['name']
kwargs['role'] = role[
'name'] #TODO: Handle multiple roles?
params = {'q': query}
params.update(kwargs)
solr_rsp = solr._select(params)
result = solr.decoder.decode(solr_rsp)
response = result.get('response') or {}
facets = result.get('facet_counts') or {}
stats = result.get('stats') or {}
grouped = result.get("grouped")
highlights = result.get("highlighting")
#app.logger.info("Facets: " + facets)
numFound = response.get('numFound', 0)
result_kwargs = process_solr_rsp(result)
results = Results(response.get('docs', ()), numFound, **result_kwargs)
page_count = int(math.ceil(numFound / 10.0))
start = response.get('start', 0)
current_page_number = int(math.ceil(start / 10.0))
if page_count > 0:
current_page_number += 1
else:
current_page_number = 1
page_count = 1
#page_count = (int) Math.ceil(results_found / (double) results_per_page);
#current_page_number = (int) Math.ceil(start / (double) results_per_page) + (page_count > 0 ? 1 : 0);
#
#app.logger.info("Saw {0} result(s).".format(len(results)))
next_start = start + 10
prev_start = max(start - 10, 0)
filter_urls = {}
if fq:
i = 0
filter_base_url = url_for('standard', start=str(start), q=query)
for outer in fq:
filter_urls[outer] = filter_base_url
for inner in fq:
if outer != inner:
app.logger.info("Inner: " + inner)
filter_urls[outer] += "&fq=" + inner
i += 1
current_url = url_for('standard',
start=str(start),
q=query,
fq=fq,
active=active)
results_url = url_for('standard',
start=str(start),
q=query,
fq=fq,
active="Results")
historical_url = url_for('standard',
start=str(start),
q=query,
fq=fq,
active="Historical")
next_url = url_for('standard',
start=str(next_start),
q=query,
fq=fq,
active=active)
prev_url = url_for('standard',
start=str(prev_start),
q=query,
fq=fq,
active=active)
app.logger.info("Next: " + next_url)
return render_template('standard.jinja2',
name=name,
search_results=results,
fq=fq,
the_user=user,
grouped=grouped,
active=active,
filter_urls=filter_urls,
raw_response=response,
start=start,
current_url=current_url,
historical_url=historical_url,
results_url=results_url,
the_facets=facets,
the_stats=stats,
the_query=query,
current_page=current_page_number,
next_url=next_url,
prev_url=prev_url,
the_page_count=page_count,
highlights=highlights,
users=users,
sort_criteria=sort_criteria)
