def search_more_term(field1,
query1,
field2,
query2,
index_name,
index_type,
kw=None,
*arg):
must1 = pyes.TermQuery(field1, query1)
must2 = pyes.TermQuery(field2, query2)
must = [must1, must2]
if arg:
must3 = pyes.TermQuery(arg[0], arg[1])
must.append(must3)
query = pyes.BoolQuery(must=must)
if kw:
search = search_add_sort(query, kw["sort_field"], kw["sort_type"])
return [i for i in CONN_ES.search(search, indices=[index_name])]
return [
i for i in CONN_ES.search(
query=query, indices=index_name, doc_types=index_type)
]
def index_object_test(self, verify=False):
"""Retrieve details for object and index it."""
config_file = config.get('config_file', '/etc/swift/swearch.conf')
self.queue_config = config.get('queue', {})
self.index_config = config.get('index', {})
indexer = mock.Mock()
account = mock.Mock()
container = mock.Mock()
objects = mock.Mock()
headers = mock.Mock()
verify = mock.Mock()
update_handler = index.ObjectUpdateHandler(indexer,
account,
container,
objects,
index=index)
self.assertTrue(update_handler)
#_id, props = update_handler.parse_props(headers)
client = index.get_search_client(self.index_config)
_id = mock.Mock() #, props = update_handler.parse_props(headers)
q = pyes.TermQuery("_id", _id)
results = client.search(
query=q,
indices=[self.index_config.get('search_index_name', 'os_default')])
def index_container(self, account, container, marker=None, verify=False):
"List all objects for the container and create task for each"
self.logger.debug("Indexing container %s/%s; marker=%s" %
(account.encode('utf8'), container.encode('utf8'),
(marker or '').encode('utf8')))
# Index the container up to marker
url = '/'.join(['http://127.0.0.1', 'v1', account])
headers, objects = get_container(url,
self.auth_token,
container,
limit=self.container_listing_count,
marker=marker,
http_conn=self._get_conn(url))
self.logger.info("Found %s Objects(s) in container %s" %
(len(objects), container.encode('utf8')))
if len(objects) == self.container_listing_count:
self.indexer.publish_backfill_task('container',
account,
container,
marker=objects[-1]['name'],
verify=verify)
update_handler = ContainerUpdateHandler(self.indexer,
account,
container,
index=self.index)
_id, props = update_handler.parse_props(headers)
if verify:
# Get Indexed State of Container
client = get_search_client(self.index_config)
results = client.search(query=pyes.TermQuery("_id", _id),
indices=[
self.index_config.get(
'search_index_name', 'os_default')
])
assert results.total == 1, '%s indexed instead of 1' % \
results.total
for name, val in results[0].iteritems():
if encode_utf8(val) != encode_utf8(props[name]):
self.logger.error('Indexed Property Does Not Match')
self.logger.error(val)
self.logger.error(props[name])
else:
update_handler.index(_id, props, block=True)
# List all of the objects in the container
self.logger.info(
"Found %s Objects in %s/%s" %
(len(objects), account.encode('utf8'), container.encode('utf8')))
for obj in objects:
self.indexer.publish_backfill_task('object',
account,
container,
obj['name'],
verify=verify)
def make_query():
qrange = pyes.RangeQuery(pyes.ESRange('utctimestamp', start_date, end_date))
qterm = pyes.TermQuery('_type', 'alert')
q = pyes.BoolQuery(must=[qrange, qterm])
results = esconn.search(q, indices=indexname)
ret = []
for x in results:
na = Alert()
if na.parse_result(x): ret.append(na)
return ret
def trefwoordview(request, trefnum):
q = request.GET.get('q')
es = pyes.ES(settings.ELASTIC_SEARCH)
results = es.search(pyes.TermQuery("trefnum", trefnum), 'nedind',
'trefwoord')
if results.count() < 1:
raise Exception('We need a 404')
return direct_to_template(request, "trefwoord.html", {
'lemma': results.next(),
'q': q
})
def execute(self):
conn = pyes.ES(self.elastic_hosts)
queries = []
filters = []
filters.append(pyes.TermFilter('account', self.account))
for field, val in self.conditions:
if val != '':
queries.append(pyes.TextQuery(field, val))
if self.type not in [None, '']:
queries.append(pyes.TextQuery('type', self.type))
if self.path not in [None, '']:
if self.recursive:
filters.append(pyes.PrefixFilter('path', '%s/' % self.path))
else:
queries.append(pyes.TermQuery('dir', self.path))
if self.marker not in [None, '']:
filters.append(
pyes.RangeFilter(pyes.ESRangeOp('name', 'gt', self.marker)))
q = pyes.MatchAllQuery()
if len(queries) > 0:
q = pyes.BoolQuery(queries)
q = pyes.FilteredQuery(q, pyes.ANDFilter(filters))
self.logger.info("Running query: %s" % q.serialize())
results = conn.search(q, self.search_index_name, start=self.start,
size=self.limit)
if self.sort not in [None, '']:
sort_list = []
for row in self.sort.split(','):
sort_data = row.split(' ')
prefix = ""
if len(sort_data) > 1 and sort_data[1].lower() == 'desc':
prefix = "-"
if sort_data[0] in SORT_WHITELIST:
sort_list.append("{0}{1}".format(prefix, sort_data[0]))
if sort_list:
results.order_by(sort_list)
return results
def index_object(self, account, container, obj, verify=False):
"Get details for object and index it"
url = '/'.join(['http://127.0.0.1', 'v1', account])
headers = head_object(url,
self.auth_token,
container,
obj,
http_conn=self._get_conn(url))
self.logger.debug("Indexing Object: %s/%s/%s" %
(account.encode('utf8'), container.encode('utf8'),
obj.encode('utf8')))
update_handler = ObjectUpdateHandler(self.indexer,
account,
container,
obj,
index=self.index)
_id, props = update_handler.parse_props(headers)
if verify:
# Check Indexed State of Object
client = get_search_client(self.index_config)
_id, props = update_handler.parse_props(headers)
q = pyes.TermQuery("_id", _id)
results = client.search(query=q,
indices=[
self.index_config.get(
'search_index_name', 'os_default')
])
assert results.total == 1, '%s indexed instead of 1' % \
results.total
for name, val in results[0].iteritems():
if encode_utf8(val) != encode_utf8(props[name]):
self.logger.error('Indexed Property Does Not Match')
self.logger.error(val)
self.logger.error(props[name])
else:
update_handler.index(_id, props, block=True)
def query_clause(index, queries, ops, fields, types):
if fields[index] == '':
search_fields = []
else:
search_fields = [fields[index]]
if (types[index] == 'term'):
# a blank term should be ignored because it prevents any results: you
# can never find a blank term
#
# TODO: add condition to deal with a query with no clauses because all have
# been ignored
if (queries[index] == ''):
return
else:
if (fields[index] != ''):
term_field = fields[index]
else:
term_field = '_all'
return pyes.TermQuery(term_field, queries[index])
else:
return pyes.StringQuery(queries[index], search_fields=search_fields)
def search_err(self, program, receiver):
start = "%s CST" % (datetime.astimezonenow() - timedelta(
hours=self.config['RANGE'])).daysstrftime('%Y/%m/%d %H:%M:%S')
end = "%s CST" % datetime.astimezonenow().strftime('%Y/%m/%d %H:%M:%S')
index = "%s_%s" % (program, datetime.astimezonenow().strftime('%Y-%m'))
q = pyes.RangeQuery(
pyes.ESRange('datetime', from_value=start, to_value=end))
q = pyes.FilteredQuery(q, pyes.TermQuery("level", "EROR"))
conn = pyes.ES(self.config['ES'])
results = conn.search(indices=index, query=q)
num = len(results)
if num == 0:
return
_content = ""
for r in results:
_content += "%s, %s<br/>" % (r['message'], r['datetime'])
title = self.config['MAIL']['MAIL_TITLE'] % program
content = "<b>Program:%s</b></b><br/>From:%s<br/>To:%s<br/><b>Total Num: %d</b><br/><br/><b>Content are:</b><br/>%s" % (
program, start, end, num, _content)
self.sendmail(receiver, title, content)
def transfer_backlog(request):
# deal with transfer mode
file_mode = False
checked_if_in_file_mode = ''
if request.GET.get('mode', '') != '':
file_mode = True
checked_if_in_file_mode = 'checked'
# get search parameters from request
queries, ops, fields, types = advanced_search.search_parameter_prep(
request)
# redirect if no search params have been set
if not 'query' in request.GET:
return helpers.redirect_with_get_params(
'components.ingest.views.transfer_backlog',
query='',
field='',
type='')
# get string of URL parameters that should be passed along when paging
search_params = advanced_search.extract_url_search_params_from_request(
request)
# set paging variables
if not file_mode:
items_per_page = 10
else:
items_per_page = 20
page = advanced_search.extract_page_number_from_url(request)
start = page * items_per_page + 1
# perform search
conn = elasticSearchFunctions.connect_and_create_index('transfers')
try:
query = advanced_search.assemble_query(
queries,
ops,
fields,
types,
must_haves=[pyes.TermQuery('status', 'backlog')])
# use all results to pull transfer facets if not in file mode
if not file_mode:
results = conn.search_raw(
query,
indices='transfers',
type='transferfile',
)
else:
# otherwise use pages results
results = conn.search_raw(query,
indices='transfers',
type='transferfile',
start=start - 1,
size=items_per_page)
except:
return HttpResponse('Error accessing index.')
# take note of facet data
file_extension_usage = results['facets']['fileExtension']['terms']
transfer_uuids = results['facets']['sipuuid']['terms']
if not file_mode:
# run through transfers to see if they've been created yet
awaiting_creation = {}
for transfer_instance in transfer_uuids:
try:
awaiting_creation[transfer_instance.
term] = transfer_awaiting_sip_creation_v2(
transfer_instance.term)
transfer = models.Transfer.objects.get(
uuid=transfer_instance.term)
transfer_basename = os.path.basename(
transfer.currentlocation[:-1])
transfer_instance.name = transfer_basename[:-37]
transfer_instance.type = transfer.type
if transfer.accessionid != None:
transfer_instance.accession = transfer.accessionid
else:
transfer_instance.accession = ''
except:
awaiting_creation[transfer_instance.term] = False
# page data
number_of_results = len(transfer_uuids)
page_data = helpers.pager(transfer_uuids, items_per_page, page + 1)
transfer_uuids = page_data['objects']
else:
# page data
number_of_results = results.hits.total
results = transfer_backlog_augment_search_results(results)
# set remaining paging variables
end, previous_page, next_page = advanced_search.paging_related_values_for_template_use(
items_per_page, page, start, number_of_results)
# make sure results is set
try:
if results:
pass
except:
results = False
form = StorageSearchForm(initial={'query': queries[0]})
return render(request, 'ingest/backlog/search.html', locals())
def march_query_tag(field, query, sub_type):
must = pyes.TermQuery("sub_type", sub_type)
should = pyes.MatchQuery(field, query)
query = pyes.BoolQuery(must=must, should=should)
return [i for i in CONN_ES.search(query=query)]
def submit(self, field, term):
q = pyes.TermQuery(field, term)
results = self.conn.search(query=q)
return results
},
'lastname': {
'index': 'not_analyzed',
'type': 'string'
},
'age': {
'index': 'not_analyzed',
'type': 'long'
}
}
conn.indices.put_mapping('man', {'properties': mapping}, ['human'])
conn.indices.put_mapping("woman", {'properties': mapping}, ["human"])
conn.index({
'firstname': 'David',
'lastname': 'White',
'age': 18
}, 'human', 'man', True)
conn.index({
'firstname': 'Suzan',
'lastname': 'Black',
'age': 28
}, 'human', 'woman', True)
q = pyes.TermQuery('firstname', 'Suzan')
q = pyes.QueryStringQuery('Suzan')
res = conn.search(query=q)
if not res:
print 'cdv'
for r in res:
print type(res)
def query_submissions(q='',
fq=None,
sort='score desc',
start=0,
docs=10,
date=None,
facets=None):
(sort_field, sort_order) = sort.split(' ')
if sort_field == 'score':
sort_field = '_score'
sort = {sort_field: {'order': sort_order}}
query = pyes.query.BoolQuery()
query.add_must(pyes.StringQuery(q, default_operator="AND"))
rest = True
x = 0
result = []
while rest:
y = fq.find(":", x)
if y == -1:
break
temp = fq[x:y]
x = y + 1
if fq[x:x + 5] == """:
y = fq.find(""", x + 5)
if y == -1:
break
result.append((temp, fq[x + 5:y]))
x = y + 6
if x > len(fq):
break
else:
y = fq.find(";", x)
if y == -1:
result.append((temp, fq[x:len(fq)]))
break
else:
result.append((temp, fq[x:y]))
x = y + 1
for sfq in result:
if sfq[0] == 'date':
(year, month) = sfq[1].split('-')
date_start = datetime.datetime(int(year), int(month), 1)
date_end = date_start + dateutil.relativedelta.relativedelta(
months=+1, seconds=-1)
query.add_must(
pyes.RangeQuery(
qrange=pyes.ESRange('date', date_start, date_end)))
else:
query.add_must(pyes.TermQuery(field=sfq[0], value=sfq[1]))
search = pyes.query.Search(query=query,
fields=[''],
start=start,
size=docs,
sort=sort)
search.facet.add_term_facet('type')
search.facet.add_term_facet('rs')
search.facet.add_term_facet('committee')
search.facet.add_date_facet(field='date', name='date', interval='month')
es = pyes.ES(app.config['ES_HOST'] + ':' + str(app.config['ES_PORT']))
es.default_indices = [app.config['ES_INDEX_NAME_PREFIX'] + '-latest']
es.refresh()
result = es.search(search, model=lambda x, y: y)
ret = {
'numhits': result.total,
'maxscore': result.max_score,
'result': [],
'facets': {}
}
if result.max_score is not None:
ret['maxscore'] = result.max_score
for key in result.facets:
ret['facets'][key] = {}
if result.facets[key]['_type'] == 'date_histogram':
for subval in result.facets[key]['entries']:
ret['facets'][key][datetime.datetime.fromtimestamp(
int(subval['time']) /
1000).strftime('%Y-%m')] = subval['count']
if result.facets[key]['_type'] == 'terms':
for subval in result.facets[key]['terms']:
ret['facets'][key][subval['term']] = subval['count']
for r in result:
ret['result'].append({'_id': str(r['_id']), 'score': r['_score']})
return ret