def search_more_term(field1, query1, field2, query2, index_name, index_type, kw=None, *arg): must1 = pyes.TermQuery(field1, query1) must2 = pyes.TermQuery(field2, query2) must = [must1, must2] if arg: must3 = pyes.TermQuery(arg[0], arg[1]) must.append(must3) query = pyes.BoolQuery(must=must) if kw: search = search_add_sort(query, kw["sort_field"], kw["sort_type"]) return [i for i in CONN_ES.search(search, indices=[index_name])] return [ i for i in CONN_ES.search( query=query, indices=index_name, doc_types=index_type) ]
def index_object_test(self, verify=False): """Retrieve details for object and index it.""" config_file = config.get('config_file', '/etc/swift/swearch.conf') self.queue_config = config.get('queue', {}) self.index_config = config.get('index', {}) indexer = mock.Mock() account = mock.Mock() container = mock.Mock() objects = mock.Mock() headers = mock.Mock() verify = mock.Mock() update_handler = index.ObjectUpdateHandler(indexer, account, container, objects, index=index) self.assertTrue(update_handler) #_id, props = update_handler.parse_props(headers) client = index.get_search_client(self.index_config) _id = mock.Mock() #, props = update_handler.parse_props(headers) q = pyes.TermQuery("_id", _id) results = client.search( query=q, indices=[self.index_config.get('search_index_name', 'os_default')])
def index_container(self, account, container, marker=None, verify=False): "List all objects for the container and create task for each" self.logger.debug("Indexing container %s/%s; marker=%s" % (account.encode('utf8'), container.encode('utf8'), (marker or '').encode('utf8'))) # Index the container up to marker url = '/'.join(['http://127.0.0.1', 'v1', account]) headers, objects = get_container(url, self.auth_token, container, limit=self.container_listing_count, marker=marker, http_conn=self._get_conn(url)) self.logger.info("Found %s Objects(s) in container %s" % (len(objects), container.encode('utf8'))) if len(objects) == self.container_listing_count: self.indexer.publish_backfill_task('container', account, container, marker=objects[-1]['name'], verify=verify) update_handler = ContainerUpdateHandler(self.indexer, account, container, index=self.index) _id, props = update_handler.parse_props(headers) if verify: # Get Indexed State of Container client = get_search_client(self.index_config) results = client.search(query=pyes.TermQuery("_id", _id), indices=[ self.index_config.get( 'search_index_name', 'os_default') ]) assert results.total == 1, '%s indexed instead of 1' % \ results.total for name, val in results[0].iteritems(): if encode_utf8(val) != encode_utf8(props[name]): self.logger.error('Indexed Property Does Not Match') self.logger.error(val) self.logger.error(props[name]) else: update_handler.index(_id, props, block=True) # List all of the objects in the container self.logger.info( "Found %s Objects in %s/%s" % (len(objects), account.encode('utf8'), container.encode('utf8'))) for obj in objects: self.indexer.publish_backfill_task('object', account, container, obj['name'], verify=verify)
def make_query(): qrange = pyes.RangeQuery(pyes.ESRange('utctimestamp', start_date, end_date)) qterm = pyes.TermQuery('_type', 'alert') q = pyes.BoolQuery(must=[qrange, qterm]) results = esconn.search(q, indices=indexname) ret = [] for x in results: na = Alert() if na.parse_result(x): ret.append(na) return ret
def trefwoordview(request, trefnum): q = request.GET.get('q') es = pyes.ES(settings.ELASTIC_SEARCH) results = es.search(pyes.TermQuery("trefnum", trefnum), 'nedind', 'trefwoord') if results.count() < 1: raise Exception('We need a 404') return direct_to_template(request, "trefwoord.html", { 'lemma': results.next(), 'q': q })
def execute(self): conn = pyes.ES(self.elastic_hosts) queries = [] filters = [] filters.append(pyes.TermFilter('account', self.account)) for field, val in self.conditions: if val != '': queries.append(pyes.TextQuery(field, val)) if self.type not in [None, '']: queries.append(pyes.TextQuery('type', self.type)) if self.path not in [None, '']: if self.recursive: filters.append(pyes.PrefixFilter('path', '%s/' % self.path)) else: queries.append(pyes.TermQuery('dir', self.path)) if self.marker not in [None, '']: filters.append( pyes.RangeFilter(pyes.ESRangeOp('name', 'gt', self.marker))) q = pyes.MatchAllQuery() if len(queries) > 0: q = pyes.BoolQuery(queries) q = pyes.FilteredQuery(q, pyes.ANDFilter(filters)) self.logger.info("Running query: %s" % q.serialize()) results = conn.search(q, self.search_index_name, start=self.start, size=self.limit) if self.sort not in [None, '']: sort_list = [] for row in self.sort.split(','): sort_data = row.split(' ') prefix = "" if len(sort_data) > 1 and sort_data[1].lower() == 'desc': prefix = "-" if sort_data[0] in SORT_WHITELIST: sort_list.append("{0}{1}".format(prefix, sort_data[0])) if sort_list: results.order_by(sort_list) return results
def index_object(self, account, container, obj, verify=False): "Get details for object and index it" url = '/'.join(['http://127.0.0.1', 'v1', account]) headers = head_object(url, self.auth_token, container, obj, http_conn=self._get_conn(url)) self.logger.debug("Indexing Object: %s/%s/%s" % (account.encode('utf8'), container.encode('utf8'), obj.encode('utf8'))) update_handler = ObjectUpdateHandler(self.indexer, account, container, obj, index=self.index) _id, props = update_handler.parse_props(headers) if verify: # Check Indexed State of Object client = get_search_client(self.index_config) _id, props = update_handler.parse_props(headers) q = pyes.TermQuery("_id", _id) results = client.search(query=q, indices=[ self.index_config.get( 'search_index_name', 'os_default') ]) assert results.total == 1, '%s indexed instead of 1' % \ results.total for name, val in results[0].iteritems(): if encode_utf8(val) != encode_utf8(props[name]): self.logger.error('Indexed Property Does Not Match') self.logger.error(val) self.logger.error(props[name]) else: update_handler.index(_id, props, block=True)
def query_clause(index, queries, ops, fields, types): if fields[index] == '': search_fields = [] else: search_fields = [fields[index]] if (types[index] == 'term'): # a blank term should be ignored because it prevents any results: you # can never find a blank term # # TODO: add condition to deal with a query with no clauses because all have # been ignored if (queries[index] == ''): return else: if (fields[index] != ''): term_field = fields[index] else: term_field = '_all' return pyes.TermQuery(term_field, queries[index]) else: return pyes.StringQuery(queries[index], search_fields=search_fields)
def search_err(self, program, receiver): start = "%s CST" % (datetime.astimezonenow() - timedelta( hours=self.config['RANGE'])).daysstrftime('%Y/%m/%d %H:%M:%S') end = "%s CST" % datetime.astimezonenow().strftime('%Y/%m/%d %H:%M:%S') index = "%s_%s" % (program, datetime.astimezonenow().strftime('%Y-%m')) q = pyes.RangeQuery( pyes.ESRange('datetime', from_value=start, to_value=end)) q = pyes.FilteredQuery(q, pyes.TermQuery("level", "EROR")) conn = pyes.ES(self.config['ES']) results = conn.search(indices=index, query=q) num = len(results) if num == 0: return _content = "" for r in results: _content += "%s, %s<br/>" % (r['message'], r['datetime']) title = self.config['MAIL']['MAIL_TITLE'] % program content = "<b>Program:%s</b></b><br/>From:%s<br/>To:%s<br/><b>Total Num: %d</b><br/><br/><b>Content are:</b><br/>%s" % ( program, start, end, num, _content) self.sendmail(receiver, title, content)
def transfer_backlog(request): # deal with transfer mode file_mode = False checked_if_in_file_mode = '' if request.GET.get('mode', '') != '': file_mode = True checked_if_in_file_mode = 'checked' # get search parameters from request queries, ops, fields, types = advanced_search.search_parameter_prep( request) # redirect if no search params have been set if not 'query' in request.GET: return helpers.redirect_with_get_params( 'components.ingest.views.transfer_backlog', query='', field='', type='') # get string of URL parameters that should be passed along when paging search_params = advanced_search.extract_url_search_params_from_request( request) # set paging variables if not file_mode: items_per_page = 10 else: items_per_page = 20 page = advanced_search.extract_page_number_from_url(request) start = page * items_per_page + 1 # perform search conn = elasticSearchFunctions.connect_and_create_index('transfers') try: query = advanced_search.assemble_query( queries, ops, fields, types, must_haves=[pyes.TermQuery('status', 'backlog')]) # use all results to pull transfer facets if not in file mode if not file_mode: results = conn.search_raw( query, indices='transfers', type='transferfile', ) else: # otherwise use pages results results = conn.search_raw(query, indices='transfers', type='transferfile', start=start - 1, size=items_per_page) except: return HttpResponse('Error accessing index.') # take note of facet data file_extension_usage = results['facets']['fileExtension']['terms'] transfer_uuids = results['facets']['sipuuid']['terms'] if not file_mode: # run through transfers to see if they've been created yet awaiting_creation = {} for transfer_instance in transfer_uuids: try: awaiting_creation[transfer_instance. term] = transfer_awaiting_sip_creation_v2( transfer_instance.term) transfer = models.Transfer.objects.get( uuid=transfer_instance.term) transfer_basename = os.path.basename( transfer.currentlocation[:-1]) transfer_instance.name = transfer_basename[:-37] transfer_instance.type = transfer.type if transfer.accessionid != None: transfer_instance.accession = transfer.accessionid else: transfer_instance.accession = '' except: awaiting_creation[transfer_instance.term] = False # page data number_of_results = len(transfer_uuids) page_data = helpers.pager(transfer_uuids, items_per_page, page + 1) transfer_uuids = page_data['objects'] else: # page data number_of_results = results.hits.total results = transfer_backlog_augment_search_results(results) # set remaining paging variables end, previous_page, next_page = advanced_search.paging_related_values_for_template_use( items_per_page, page, start, number_of_results) # make sure results is set try: if results: pass except: results = False form = StorageSearchForm(initial={'query': queries[0]}) return render(request, 'ingest/backlog/search.html', locals())
def march_query_tag(field, query, sub_type): must = pyes.TermQuery("sub_type", sub_type) should = pyes.MatchQuery(field, query) query = pyes.BoolQuery(must=must, should=should) return [i for i in CONN_ES.search(query=query)]
def submit(self, field, term): q = pyes.TermQuery(field, term) results = self.conn.search(query=q) return results
}, 'lastname': { 'index': 'not_analyzed', 'type': 'string' }, 'age': { 'index': 'not_analyzed', 'type': 'long' } } conn.indices.put_mapping('man', {'properties': mapping}, ['human']) conn.indices.put_mapping("woman", {'properties': mapping}, ["human"]) conn.index({ 'firstname': 'David', 'lastname': 'White', 'age': 18 }, 'human', 'man', True) conn.index({ 'firstname': 'Suzan', 'lastname': 'Black', 'age': 28 }, 'human', 'woman', True) q = pyes.TermQuery('firstname', 'Suzan') q = pyes.QueryStringQuery('Suzan') res = conn.search(query=q) if not res: print 'cdv' for r in res: print type(res)
def query_submissions(q='', fq=None, sort='score desc', start=0, docs=10, date=None, facets=None): (sort_field, sort_order) = sort.split(' ') if sort_field == 'score': sort_field = '_score' sort = {sort_field: {'order': sort_order}} query = pyes.query.BoolQuery() query.add_must(pyes.StringQuery(q, default_operator="AND")) rest = True x = 0 result = [] while rest: y = fq.find(":", x) if y == -1: break temp = fq[x:y] x = y + 1 if fq[x:x + 5] == """: y = fq.find(""", x + 5) if y == -1: break result.append((temp, fq[x + 5:y])) x = y + 6 if x > len(fq): break else: y = fq.find(";", x) if y == -1: result.append((temp, fq[x:len(fq)])) break else: result.append((temp, fq[x:y])) x = y + 1 for sfq in result: if sfq[0] == 'date': (year, month) = sfq[1].split('-') date_start = datetime.datetime(int(year), int(month), 1) date_end = date_start + dateutil.relativedelta.relativedelta( months=+1, seconds=-1) query.add_must( pyes.RangeQuery( qrange=pyes.ESRange('date', date_start, date_end))) else: query.add_must(pyes.TermQuery(field=sfq[0], value=sfq[1])) search = pyes.query.Search(query=query, fields=[''], start=start, size=docs, sort=sort) search.facet.add_term_facet('type') search.facet.add_term_facet('rs') search.facet.add_term_facet('committee') search.facet.add_date_facet(field='date', name='date', interval='month') es = pyes.ES(app.config['ES_HOST'] + ':' + str(app.config['ES_PORT'])) es.default_indices = [app.config['ES_INDEX_NAME_PREFIX'] + '-latest'] es.refresh() result = es.search(search, model=lambda x, y: y) ret = { 'numhits': result.total, 'maxscore': result.max_score, 'result': [], 'facets': {} } if result.max_score is not None: ret['maxscore'] = result.max_score for key in result.facets: ret['facets'][key] = {} if result.facets[key]['_type'] == 'date_histogram': for subval in result.facets[key]['entries']: ret['facets'][key][datetime.datetime.fromtimestamp( int(subval['time']) / 1000).strftime('%Y-%m')] = subval['count'] if result.facets[key]['_type'] == 'terms': for subval in result.facets[key]['terms']: ret['facets'][key][subval['term']] = subval['count'] for r in result: ret['result'].append({'_id': str(r['_id']), 'score': r['_score']}) return ret