def query(self, solr_query, hue_core):
try:
params = self._get_params() + (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
try:
response = json.loads(response)
except ValueError, e:
# Got some null bytes in the response
LOG.error('%s: %s' % (unicode(e), repr(response)))
response = json.loads(response.replace('\x00', ''))
return response
def query(self, solr_query, hue_core):
try:
params = (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq), )
response = self._root.get('%(collection)s/select' % solr_query,
params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
response = json.loads(response)
return response
except RestException, e:
raise PopupException('Error while accessing Solr: %s' % e)
def query(self, solr_query, hue_core):
try:
params = self._get_params() + (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
try:
response = json.loads(response)
except ValueError, e:
# Got some null bytes in the response
LOG.error('%s: %s' % (unicode(e), repr(response)))
response = json.loads(response.replace('\x00', ''))
return response
def stats(self, core, fields):
try:
params = (
('q', EMPTY_QUERY.get()),
('wt', 'json'),
('rows', 0),
('stats', 'true'),
)
params += tuple([('stats.field', field) for field in fields])
response = self._root.get('%(core)s/select' % {'core': core}, params=params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def select(self, collection, query=None, rows=100, start=0):
if query is None:
query = EMPTY_QUERY.get()
params = self._get_params() + (
('q', query),
('wt', 'json'),
('rows', rows),
('start', start),
)
response = self._root.get('%s/select' % collection, params)
return self._get_json(response)
def stats(self, core, fields):
try:
params = self._get_params() + (
('q', EMPTY_QUERY.get()),
('wt', 'json'),
('rows', 0),
('stats', 'true'),
)
params += tuple([('stats.field', field) for field in fields])
response = self._root.get('%(core)s/select' % {'core': core},
params=params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def stats(self, core, fields, query=None, facet=''):
try:
params = self._get_params() + (
('q', self._get_q(query) if query is not None else EMPTY_QUERY.get()),
('wt', 'json'),
('rows', 0),
('stats', 'true'),
)
if facet:
params += (('stats.facet', facet),)
params += tuple([('stats.field', field) for field in fields])
response = self._root.get('%(core)s/select' % {'core': core}, params=params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def query(self, solr_query, hue_core):
try:
params = (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query()
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
return json.loads(response)
except RestException, e:
raise PopupException('Error while accessing Solr: %s' % e)
def stats(self, core, fields, query=None, facet=''):
try:
params = self._get_params() + (
('q', self._get_q(query) if query is not None else EMPTY_QUERY.get()),
('wt', 'json'),
('rows', 0),
('stats', 'true'),
)
if query is not None:
params += self._get_fq(query)
if facet:
params += (('stats.facet', facet),)
params += tuple([('stats.field', field) for field in fields])
response = self._root.get('%(core)s/select' % {'core': core}, params=params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def query(self, solr_query, hue_core):
try:
params = self._get_params() + (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', urllib.unquote(utf_quoter(fq))),)
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def stats(self, core, fields, query=None, facet=""):
try:
params = self._get_params() + (
("q", self._get_q(query) if query is not None else EMPTY_QUERY.get()),
("wt", "json"),
("rows", 0),
("stats", "true"),
)
if query is not None:
params += self._get_fq(None, query)
if facet:
params += (("stats.facet", facet),)
params += tuple([("stats.field", field) for field in fields])
response = self._root.get("%(core)s/select" % {"core": core}, params=params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_("Error while accessing Solr"))
def query(self, solr_query, hue_core):
try:
params = self._get_params() + (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', urllib.unquote(utf_quoter(fq))), )
response = self._root.get('%(collection)s/select' % solr_query,
params)
return self._get_json(response)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def query(self, solr_query, hue_core):
try:
params = self._get_params() + (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
response = json.loads(response)
return response
except RestException, e:
raise PopupException('Error while accessing Solr: %s' % e)
def query(self, solr_query, hue_core):
try:
params = (
("q", solr_query["q"] or EMPTY_QUERY.get()),
("wt", "json"),
("rows", solr_query["rows"]),
("start", solr_query["start"]),
)
params += hue_core.get_query(solr_query)
fqs = solr_query["fq"].split("|")
for fq in fqs:
if fq:
params += (("fq", fq),)
response = self._root.get("%(collection)s/select" % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
response = json.loads(response)
return response
except RestException, e:
raise PopupException("Error while accessing Solr: %s" % e)
def _get_q(self, query):
q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
def query(self, collection, query):
solr_query = {}
solr_query['collection'] = collection['name']
if query.get('download'):
solr_query['rows'] = 1000
solr_query['start'] = 0
else:
solr_query['rows'] = int(collection['template']['rows'] or 10)
solr_query['start'] = int(query['start'])
solr_query['rows'] = min(solr_query['rows'], 1000)
solr_query['start'] = min(solr_query['start'], 10000)
params = self._get_params() + (
('q', self._get_q(query)),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
if any(collection['facets']):
params += (
('facet', 'true'),
('facet.mincount', 0),
('facet.limit', 10),
)
json_facets = {}
timeFilter = self._get_range_borders(collection, query)
for facet in collection['facets']:
if facet['type'] == 'query':
params += (('facet.query', '%s' % facet['field']),)
elif facet['type'] == 'range' or facet['type'] == 'range-up':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'start': facet['properties']['start'],
'end': facet['properties']['end'],
'gap': facet['properties']['gap'],
'mincount': int(facet['properties']['mincount'])
}
if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'):
keys.update(self._get_time_filter_query(timeFilter, facet))
params += (
('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
)
elif facet['type'] == 'field':
keys = {
'id': '%(id)s' % facet,
'field': facet['field'],
'key': '%(field)s-%(id)s' % facet,
'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0),
'mincount': int(facet['properties']['mincount'])
}
params += (
('facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys),
)
elif facet['type'] == 'nested':
_f = {}
if facet['properties']['facets']:
self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1, timeFilter)
if facet['properties'].get('domain'):
if facet['properties']['domain'].get('blockParent') or facet['properties']['domain'].get('blockChildren'):
_f['domain'] = {}
if facet['properties']['domain'].get('blockParent'):
_f['domain']['blockParent'] = ' OR '.join(facet['properties']['domain']['blockParent'])
if facet['properties']['domain'].get('blockChildren'):
_f['domain']['blockChildren'] = ' OR '.join(facet['properties']['domain']['blockChildren'])
if _f:
sort = {'count': facet['properties']['facets'][0]['sort']}
for i, agg in enumerate(self._get_dimension_aggregates(facet['properties']['facets'][1:])):
if agg['sort'] != 'default':
agg_function = self._get_aggregate_function(agg)
sort = {'agg_%02d_%02d:%s' % (1, i, agg_function): agg['sort']}
if sort.get('count') == 'default':
sort['count'] = 'desc'
dim_key = [key for key in _f['facet'].keys() if 'dim' in key][0]
_f['facet'][dim_key].update({
'excludeTags': facet['id'],
'offset': 0,
'numBuckets': True,
'allBuckets': True,
'sort': sort
#'prefix': '' # Forbidden on numeric fields
})
json_facets[facet['id']] = _f['facet'][dim_key]
elif facet['type'] == 'function':
if facet['properties']['facets']:
json_facets[facet['id']] = self._get_aggregate_function(facet['properties']['facets'][0])
if facet['properties']['compare']['is_enabled']:
# TODO: global compare override
unit = re.split('\d+', facet['properties']['compare']['gap'])[1]
json_facets[facet['id']] = {
'type': 'range',
'field': collection['timeFilter'].get('field'),
'start': 'NOW/%s-%s-%s' % (unit, facet['properties']['compare']['gap'], facet['properties']['compare']['gap']),
'end': 'NOW/%s' % unit,
'gap': '+%(gap)s' % facet['properties']['compare'],
'facet': {facet['id']: json_facets[facet['id']]}
}
if facet['properties']['filter']['is_enabled']:
json_facets[facet['id']] = {
'type': 'query',
'q': facet['properties']['filter']['query'] or EMPTY_QUERY.get(),
'facet': {facet['id']: json_facets[facet['id']]}
}
json_facets['processEmpty'] = True
elif facet['type'] == 'pivot':
if facet['properties']['facets'] or facet['widgetType'] == 'map-widget':
fields = facet['field']
fields_limits = []
for f in facet['properties']['facets']:
fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit']))
fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount']))
fields += ',' + f['field']
keys = {
'id': '%(id)s' % facet,
'key': '%(field)s-%(id)s' % facet,
'field': facet['field'],
'fields': fields,
'limit': int(facet['properties'].get('limit', 10)),
'mincount': int(facet['properties']['mincount']),
'fields_limits': ' '.join(fields_limits)
}
params += (
('facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys),
)
if json_facets:
params += (
('json.facet', json.dumps(json_facets)),
)
params += self._get_fq(collection, query)
fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection))))
nested_fields = self._get_nested_fields(collection)
if nested_fields:
fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields)))
if collection['template']['moreLikeThis'] and fl != ['*']: # Potential conflict with nested documents
id_field = collection.get('idField', 'id')
params += (
('mlt', 'true'),
('mlt.fl', fl.replace(',%s' % id_field, '')),
('mlt.mintf', 1),
('mlt.mindf', 1),
('mlt.maxdf', 50),
('mlt.maxntp', 1000),
('mlt.count', 10),
#('mlt.minwl', 1),
#('mlt.maxwl', 1),
)
fl = '*'
params += (('fl', fl),)
params += (
('hl', 'true'),
('hl.fl', '*'),
('hl.snippets', 5),
('hl.fragsize', 1000),
)
if collection['template']['fieldsSelected']:
fields = []
for field in collection['template']['fieldsSelected']:
attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
if attribute_field:
if attribute_field[0]['sort']['direction']:
fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
if fields:
params += (
('sort', ','.join(fields)),
)
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
def query(self, collection, query):
solr_query = {}
solr_query['collection'] = collection['name']
if query.get('download'):
solr_query['rows'] = 1000
solr_query['start'] = 0
else:
solr_query['rows'] = int(collection['template']['rows'] or 10)
solr_query['start'] = int(query['start'])
solr_query['rows'] = min(solr_query['rows'], 1000)
solr_query['start'] = min(solr_query['start'], 10000)
q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
params = self._get_params() + (
('q', 'OR'.join([
q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']
])),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
if any(collection['facets']):
params += (
('facet', 'true'),
('facet.mincount', 0),
('facet.limit', 10),
)
for facet in collection['facets']:
if facet['type'] == 'query':
params += (('facet.query', '%s' % facet['field']), )
elif facet['type'] == 'range':
params += tuple([
('facet.range',
'{!ex=%s}%s' % (facet['field'], facet['field'])),
('f.%s.facet.range.start' % facet['field'],
facet['properties']['start']),
('f.%s.facet.range.end' % facet['field'],
facet['properties']['end']),
('f.%s.facet.range.gap' % facet['field'],
facet['properties']['gap']),
('f.%s.facet.mincount' % facet['field'],
facet['properties']['mincount']),
])
elif facet['type'] == 'field':
params += (
('facet.field',
'{!ex=%s}%s' % (facet['field'], facet['field'])),
('f.%s.facet.limit' % facet['field'],
int(facet['properties'].get('limit', 10)) + 1),
('f.%s.facet.mincount' % facet['field'],
int(facet['properties']['mincount'])),
)
for fq in query['fqs']:
if fq['type'] == 'field':
# This does not work if spaces in Solr:
# params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),)
f = []
for _filter in fq['filter']:
if _filter is not None and ' ' in _filter:
f.append('%s:"%s"' % (fq['field'], _filter))
else:
f.append('{!field f=%s}%s' % (fq['field'], _filter))
params += (('fq',
urllib.unquote(
utf_quoter('{!tag=%s}' % fq['field'] +
' '.join(f)))), )
elif fq['type'] == 'range':
params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([
urllib.unquote(
utf_quoter('%s:[%s TO %s}' %
(fq['field'], f['from'], f['to'])))
for f in fq['properties']
])), )
if collection['template']['fieldsSelected'] and collection['template'][
'isGridLayout']:
fields = collection['template']['fieldsSelected'] + [
collection['idField']
] if collection['idField'] else []
params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))), )
else:
params += (('fl', '*'), )
params += (('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 3))
if collection['template']['fieldsSelected']:
fields = []
for field in collection['template']['fieldsSelected']:
attribute_field = filter(
lambda attribute: field == attribute['name'],
collection['template']['fieldsAttributes'])
if attribute_field:
if attribute_field[0]['sort']['direction']:
fields.append(
'%s %s' %
(field, attribute_field[0]['sort']['direction']))
if fields:
params += (('sort', ','.join(fields)), )
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
def query(self, collection, query):
solr_query = {}
solr_query['collection'] = collection['name']
if query.get('download'):
solr_query['rows'] = 1000
solr_query['start'] = 0
else:
solr_query['rows'] = int(collection['template']['rows'] or 10)
solr_query['start'] = int(query['start'])
solr_query['rows'] = min(solr_query['rows'], 1000)
solr_query['start'] = min(solr_query['start'], 10000)
q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
params = self._get_params() + (
('q', 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']])),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
if any(collection['facets']):
params += (
('facet', 'true'),
('facet.mincount', 0),
('facet.limit', 10),
)
for facet in collection['facets']:
if facet['type'] == 'query':
params += (('facet.query', '%s' % facet['field']),)
elif facet['type'] == 'range':
params += tuple([
('facet.range', '{!ex=%s}%s' % (facet['field'], facet['field'])),
('f.%s.facet.range.start' % facet['field'], facet['properties']['start']),
('f.%s.facet.range.end' % facet['field'], facet['properties']['end']),
('f.%s.facet.range.gap' % facet['field'], facet['properties']['gap']),
('f.%s.facet.mincount' % facet['field'], facet['properties']['mincount']),]
)
elif facet['type'] == 'field':
params += (
('facet.field', '{!ex=%s}%s' % (facet['field'], facet['field'])),
('f.%s.facet.limit' % facet['field'], int(facet['properties'].get('limit', 10)) + 1),
('f.%s.facet.mincount' % facet['field'], int(facet['properties']['mincount'])),
)
for fq in query['fqs']:
if fq['type'] == 'field':
# This does not work if spaces in Solr:
# params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),)
f = []
for _filter in fq['filter']:
if _filter is not None and ' ' in _filter:
f.append('%s:"%s"' % (fq['field'], _filter))
else:
f.append('{!field f=%s}%s' % (fq['field'], _filter))
params += (('fq', urllib.unquote(utf_quoter('{!tag=%s}' % fq['field'] + ' '.join(f)))),)
elif fq['type'] == 'range':
params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([urllib.unquote(utf_quoter('%s:[%s TO %s}' % (fq['field'], f['from'], f['to']))) for f in fq['properties']])),)
if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']:
fields = collection['template']['fieldsSelected'] + [collection['idField']] if collection['idField'] else []
params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))),)
else:
params += (('fl', '*'),)
params += (
('hl', 'true'),
('hl.fl', '*'),
('hl.snippets', 3)
)
if collection['template']['fieldsSelected']:
fields = []
for field in collection['template']['fieldsSelected']:
attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes'])
if attribute_field:
if attribute_field[0]['sort']['direction']:
fields.append('%s %s' % (field, attribute_field[0]['sort']['direction']))
if fields:
params += (
('sort', ','.join(fields)),
)
response = self._root.get('%(collection)s/select' % solr_query, params)
return self._get_json(response)
def _get_q(self, query):
q_template = '(%s)' if len(query['qs']) >= 2 else '%s'
return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
def _get_q(self, query):
q_template = "(%s)" if len(query["qs"]) >= 2 else "%s"
return "OR".join([q_template % (q["q"] or EMPTY_QUERY.get()) for q in query["qs"]]).encode("utf-8")
