def query(self, solr_query, hue_core): try: params = self._get_params() + ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', fq),) response = self._root.get('%(collection)s/select' % solr_query, params) if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' try: response = json.loads(response) except ValueError, e: # Got some null bytes in the response LOG.error('%s: %s' % (unicode(e), repr(response))) response = json.loads(response.replace('\x00', '')) return response
def query(self, solr_query, hue_core): try: params = ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', fq), ) response = self._root.get('%(collection)s/select' % solr_query, params) if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' response = json.loads(response) return response except RestException, e: raise PopupException('Error while accessing Solr: %s' % e)
def stats(self, core, fields): try: params = ( ('q', EMPTY_QUERY.get()), ('wt', 'json'), ('rows', 0), ('stats', 'true'), ) params += tuple([('stats.field', field) for field in fields]) response = self._root.get('%(core)s/select' % {'core': core}, params=params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def select(self, collection, query=None, rows=100, start=0): if query is None: query = EMPTY_QUERY.get() params = self._get_params() + ( ('q', query), ('wt', 'json'), ('rows', rows), ('start', start), ) response = self._root.get('%s/select' % collection, params) return self._get_json(response)
def stats(self, core, fields): try: params = self._get_params() + ( ('q', EMPTY_QUERY.get()), ('wt', 'json'), ('rows', 0), ('stats', 'true'), ) params += tuple([('stats.field', field) for field in fields]) response = self._root.get('%(core)s/select' % {'core': core}, params=params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def stats(self, core, fields, query=None, facet=''): try: params = self._get_params() + ( ('q', self._get_q(query) if query is not None else EMPTY_QUERY.get()), ('wt', 'json'), ('rows', 0), ('stats', 'true'), ) if facet: params += (('stats.facet', facet),) params += tuple([('stats.field', field) for field in fields]) response = self._root.get('%(core)s/select' % {'core': core}, params=params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def query(self, solr_query, hue_core): try: params = ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query() fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', fq),) response = self._root.get('%(collection)s/select' % solr_query, params) return json.loads(response) except RestException, e: raise PopupException('Error while accessing Solr: %s' % e)
def stats(self, core, fields, query=None, facet=''): try: params = self._get_params() + ( ('q', self._get_q(query) if query is not None else EMPTY_QUERY.get()), ('wt', 'json'), ('rows', 0), ('stats', 'true'), ) if query is not None: params += self._get_fq(query) if facet: params += (('stats.facet', facet),) params += tuple([('stats.field', field) for field in fields]) response = self._root.get('%(core)s/select' % {'core': core}, params=params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def query(self, solr_query, hue_core): try: params = self._get_params() + ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', urllib.unquote(utf_quoter(fq))),) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def stats(self, core, fields, query=None, facet=""): try: params = self._get_params() + ( ("q", self._get_q(query) if query is not None else EMPTY_QUERY.get()), ("wt", "json"), ("rows", 0), ("stats", "true"), ) if query is not None: params += self._get_fq(None, query) if facet: params += (("stats.facet", facet),) params += tuple([("stats.field", field) for field in fields]) response = self._root.get("%(core)s/select" % {"core": core}, params=params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_("Error while accessing Solr"))
def query(self, solr_query, hue_core): try: params = self._get_params() + ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', urllib.unquote(utf_quoter(fq))), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response) except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def query(self, solr_query, hue_core): try: params = self._get_params() + ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', fq),) response = self._root.get('%(collection)s/select' % solr_query, params) if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' response = json.loads(response) return response except RestException, e: raise PopupException('Error while accessing Solr: %s' % e)
def query(self, solr_query, hue_core): try: params = ( ("q", solr_query["q"] or EMPTY_QUERY.get()), ("wt", "json"), ("rows", solr_query["rows"]), ("start", solr_query["start"]), ) params += hue_core.get_query(solr_query) fqs = solr_query["fq"].split("|") for fq in fqs: if fq: params += (("fq", fq),) response = self._root.get("%(collection)s/select" % solr_query, params) if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' response = json.loads(response) return response except RestException, e: raise PopupException("Error while accessing Solr: %s" % e)
def _get_q(self, query): q_template = '(%s)' if len(query['qs']) >= 2 else '%s' return 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']]).encode('utf-8')
def query(self, collection, query): solr_query = {} solr_query['collection'] = collection['name'] if query.get('download'): solr_query['rows'] = 1000 solr_query['start'] = 0 else: solr_query['rows'] = int(collection['template']['rows'] or 10) solr_query['start'] = int(query['start']) solr_query['rows'] = min(solr_query['rows'], 1000) solr_query['start'] = min(solr_query['start'], 10000) params = self._get_params() + ( ('q', self._get_q(query)), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) if any(collection['facets']): params += ( ('facet', 'true'), ('facet.mincount', 0), ('facet.limit', 10), ) json_facets = {} timeFilter = self._get_range_borders(collection, query) for facet in collection['facets']: if facet['type'] == 'query': params += (('facet.query', '%s' % facet['field']),) elif facet['type'] == 'range' or facet['type'] == 'range-up': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'start': facet['properties']['start'], 'end': facet['properties']['end'], 'gap': facet['properties']['gap'], 'mincount': int(facet['properties']['mincount']) } if timeFilter and timeFilter['time_field'] == facet['field'] and (facet['id'] not in timeFilter['time_filter_overrides'] or facet['widgetType'] != 'histogram-widget'): keys.update(self._get_time_filter_query(timeFilter, facet)) params += ( ('facet.range', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.range.start=%(start)s f.%(field)s.facet.range.end=%(end)s f.%(field)s.facet.range.gap=%(gap)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'field': keys = { 'id': '%(id)s' % facet, 'field': facet['field'], 'key': '%(field)s-%(id)s' % facet, 'limit': int(facet['properties'].get('limit', 10)) + (1 if facet['widgetType'] == 'facet-widget' else 0), 'mincount': int(facet['properties']['mincount']) } params += ( ('facet.field', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s}%(field)s' % keys), ) elif facet['type'] == 'nested': _f = {} if facet['properties']['facets']: self._n_facet_dimension(facet, _f, facet['properties']['facets'], 1, timeFilter) if facet['properties'].get('domain'): if facet['properties']['domain'].get('blockParent') or facet['properties']['domain'].get('blockChildren'): _f['domain'] = {} if facet['properties']['domain'].get('blockParent'): _f['domain']['blockParent'] = ' OR '.join(facet['properties']['domain']['blockParent']) if facet['properties']['domain'].get('blockChildren'): _f['domain']['blockChildren'] = ' OR '.join(facet['properties']['domain']['blockChildren']) if _f: sort = {'count': facet['properties']['facets'][0]['sort']} for i, agg in enumerate(self._get_dimension_aggregates(facet['properties']['facets'][1:])): if agg['sort'] != 'default': agg_function = self._get_aggregate_function(agg) sort = {'agg_%02d_%02d:%s' % (1, i, agg_function): agg['sort']} if sort.get('count') == 'default': sort['count'] = 'desc' dim_key = [key for key in _f['facet'].keys() if 'dim' in key][0] _f['facet'][dim_key].update({ 'excludeTags': facet['id'], 'offset': 0, 'numBuckets': True, 'allBuckets': True, 'sort': sort #'prefix': '' # Forbidden on numeric fields }) json_facets[facet['id']] = _f['facet'][dim_key] elif facet['type'] == 'function': if facet['properties']['facets']: json_facets[facet['id']] = self._get_aggregate_function(facet['properties']['facets'][0]) if facet['properties']['compare']['is_enabled']: # TODO: global compare override unit = re.split('\d+', facet['properties']['compare']['gap'])[1] json_facets[facet['id']] = { 'type': 'range', 'field': collection['timeFilter'].get('field'), 'start': 'NOW/%s-%s-%s' % (unit, facet['properties']['compare']['gap'], facet['properties']['compare']['gap']), 'end': 'NOW/%s' % unit, 'gap': '+%(gap)s' % facet['properties']['compare'], 'facet': {facet['id']: json_facets[facet['id']]} } if facet['properties']['filter']['is_enabled']: json_facets[facet['id']] = { 'type': 'query', 'q': facet['properties']['filter']['query'] or EMPTY_QUERY.get(), 'facet': {facet['id']: json_facets[facet['id']]} } json_facets['processEmpty'] = True elif facet['type'] == 'pivot': if facet['properties']['facets'] or facet['widgetType'] == 'map-widget': fields = facet['field'] fields_limits = [] for f in facet['properties']['facets']: fields_limits.append('f.%s.facet.limit=%s' % (f['field'], f['limit'])) fields_limits.append('f.%s.facet.mincount=%s' % (f['field'], f['mincount'])) fields += ',' + f['field'] keys = { 'id': '%(id)s' % facet, 'key': '%(field)s-%(id)s' % facet, 'field': facet['field'], 'fields': fields, 'limit': int(facet['properties'].get('limit', 10)), 'mincount': int(facet['properties']['mincount']), 'fields_limits': ' '.join(fields_limits) } params += ( ('facet.pivot', '{!key=%(key)s ex=%(id)s f.%(field)s.facet.limit=%(limit)s f.%(field)s.facet.mincount=%(mincount)s %(fields_limits)s}%(fields)s' % keys), ) if json_facets: params += ( ('json.facet', json.dumps(json_facets)), ) params += self._get_fq(collection, query) fl = urllib.unquote(utf_quoter(','.join(Collection2.get_field_list(collection)))) nested_fields = self._get_nested_fields(collection) if nested_fields: fl += urllib.unquote(utf_quoter(',[child parentFilter="%s"]' % ' OR '.join(nested_fields))) if collection['template']['moreLikeThis'] and fl != ['*']: # Potential conflict with nested documents id_field = collection.get('idField', 'id') params += ( ('mlt', 'true'), ('mlt.fl', fl.replace(',%s' % id_field, '')), ('mlt.mintf', 1), ('mlt.mindf', 1), ('mlt.maxdf', 50), ('mlt.maxntp', 1000), ('mlt.count', 10), #('mlt.minwl', 1), #('mlt.maxwl', 1), ) fl = '*' params += (('fl', fl),) params += ( ('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 5), ('hl.fragsize', 1000), ) if collection['template']['fieldsSelected']: fields = [] for field in collection['template']['fieldsSelected']: attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes']) if attribute_field: if attribute_field[0]['sort']['direction']: fields.append('%s %s' % (field, attribute_field[0]['sort']['direction'])) if fields: params += ( ('sort', ','.join(fields)), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response)
def query(self, collection, query): solr_query = {} solr_query['collection'] = collection['name'] if query.get('download'): solr_query['rows'] = 1000 solr_query['start'] = 0 else: solr_query['rows'] = int(collection['template']['rows'] or 10) solr_query['start'] = int(query['start']) solr_query['rows'] = min(solr_query['rows'], 1000) solr_query['start'] = min(solr_query['start'], 10000) q_template = '(%s)' if len(query['qs']) >= 2 else '%s' params = self._get_params() + ( ('q', 'OR'.join([ q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs'] ])), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) if any(collection['facets']): params += ( ('facet', 'true'), ('facet.mincount', 0), ('facet.limit', 10), ) for facet in collection['facets']: if facet['type'] == 'query': params += (('facet.query', '%s' % facet['field']), ) elif facet['type'] == 'range': params += tuple([ ('facet.range', '{!ex=%s}%s' % (facet['field'], facet['field'])), ('f.%s.facet.range.start' % facet['field'], facet['properties']['start']), ('f.%s.facet.range.end' % facet['field'], facet['properties']['end']), ('f.%s.facet.range.gap' % facet['field'], facet['properties']['gap']), ('f.%s.facet.mincount' % facet['field'], facet['properties']['mincount']), ]) elif facet['type'] == 'field': params += ( ('facet.field', '{!ex=%s}%s' % (facet['field'], facet['field'])), ('f.%s.facet.limit' % facet['field'], int(facet['properties'].get('limit', 10)) + 1), ('f.%s.facet.mincount' % facet['field'], int(facet['properties']['mincount'])), ) for fq in query['fqs']: if fq['type'] == 'field': # This does not work if spaces in Solr: # params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),) f = [] for _filter in fq['filter']: if _filter is not None and ' ' in _filter: f.append('%s:"%s"' % (fq['field'], _filter)) else: f.append('{!field f=%s}%s' % (fq['field'], _filter)) params += (('fq', urllib.unquote( utf_quoter('{!tag=%s}' % fq['field'] + ' '.join(f)))), ) elif fq['type'] == 'range': params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([ urllib.unquote( utf_quoter('%s:[%s TO %s}' % (fq['field'], f['from'], f['to']))) for f in fq['properties'] ])), ) if collection['template']['fieldsSelected'] and collection['template'][ 'isGridLayout']: fields = collection['template']['fieldsSelected'] + [ collection['idField'] ] if collection['idField'] else [] params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))), ) else: params += (('fl', '*'), ) params += (('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 3)) if collection['template']['fieldsSelected']: fields = [] for field in collection['template']['fieldsSelected']: attribute_field = filter( lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes']) if attribute_field: if attribute_field[0]['sort']['direction']: fields.append( '%s %s' % (field, attribute_field[0]['sort']['direction'])) if fields: params += (('sort', ','.join(fields)), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response)
def query(self, collection, query): solr_query = {} solr_query['collection'] = collection['name'] if query.get('download'): solr_query['rows'] = 1000 solr_query['start'] = 0 else: solr_query['rows'] = int(collection['template']['rows'] or 10) solr_query['start'] = int(query['start']) solr_query['rows'] = min(solr_query['rows'], 1000) solr_query['start'] = min(solr_query['start'], 10000) q_template = '(%s)' if len(query['qs']) >= 2 else '%s' params = self._get_params() + ( ('q', 'OR'.join([q_template % (q['q'] or EMPTY_QUERY.get()) for q in query['qs']])), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) if any(collection['facets']): params += ( ('facet', 'true'), ('facet.mincount', 0), ('facet.limit', 10), ) for facet in collection['facets']: if facet['type'] == 'query': params += (('facet.query', '%s' % facet['field']),) elif facet['type'] == 'range': params += tuple([ ('facet.range', '{!ex=%s}%s' % (facet['field'], facet['field'])), ('f.%s.facet.range.start' % facet['field'], facet['properties']['start']), ('f.%s.facet.range.end' % facet['field'], facet['properties']['end']), ('f.%s.facet.range.gap' % facet['field'], facet['properties']['gap']), ('f.%s.facet.mincount' % facet['field'], facet['properties']['mincount']),] ) elif facet['type'] == 'field': params += ( ('facet.field', '{!ex=%s}%s' % (facet['field'], facet['field'])), ('f.%s.facet.limit' % facet['field'], int(facet['properties'].get('limit', 10)) + 1), ('f.%s.facet.mincount' % facet['field'], int(facet['properties']['mincount'])), ) for fq in query['fqs']: if fq['type'] == 'field': # This does not work if spaces in Solr: # params += (('fq', ' '.join([urllib.unquote(utf_quoter('{!tag=%s}{!field f=%s}%s' % (fq['field'], fq['field'], _filter))) for _filter in fq['filter']])),) f = [] for _filter in fq['filter']: if _filter is not None and ' ' in _filter: f.append('%s:"%s"' % (fq['field'], _filter)) else: f.append('{!field f=%s}%s' % (fq['field'], _filter)) params += (('fq', urllib.unquote(utf_quoter('{!tag=%s}' % fq['field'] + ' '.join(f)))),) elif fq['type'] == 'range': params += (('fq', '{!tag=%s}' % fq['field'] + ' '.join([urllib.unquote(utf_quoter('%s:[%s TO %s}' % (fq['field'], f['from'], f['to']))) for f in fq['properties']])),) if collection['template']['fieldsSelected'] and collection['template']['isGridLayout']: fields = collection['template']['fieldsSelected'] + [collection['idField']] if collection['idField'] else [] params += (('fl', urllib.unquote(utf_quoter(','.join(fields)))),) else: params += (('fl', '*'),) params += ( ('hl', 'true'), ('hl.fl', '*'), ('hl.snippets', 3) ) if collection['template']['fieldsSelected']: fields = [] for field in collection['template']['fieldsSelected']: attribute_field = filter(lambda attribute: field == attribute['name'], collection['template']['fieldsAttributes']) if attribute_field: if attribute_field[0]['sort']['direction']: fields.append('%s %s' % (field, attribute_field[0]['sort']['direction'])) if fields: params += ( ('sort', ','.join(fields)), ) response = self._root.get('%(collection)s/select' % solr_query, params) return self._get_json(response)
def _get_q(self, query): q_template = "(%s)" if len(query["qs"]) >= 2 else "%s" return "OR".join([q_template % (q["q"] or EMPTY_QUERY.get()) for q in query["qs"]]).encode("utf-8")