def get_es_conn(es_url, index):
"""Create connection and create index if it doesn't exist."""
conn = ES(es_url)
if not conn.indices.exists_index(index):
conn.indices.create_index(index)
return conn
def __init__(self, serverInfo, proto="http"):
#serverInfo can be a json object
#only connect pyes to master es node
#in the case that other nodes are taken down
#because http requests will fail
# TODO: dynamic master node detection
if isinstance(serverInfo, dict):
self.ip = serverInfo["ip"]
self.rest_username = serverInfo["username"]
self.rest_password = serverInfo["password"]
self.username = serverInfo["es_username"]
self.password = serverInfo["es_password"]
self.port = 9091 #serverInfo["port"]
else:
self.ip = serverInfo.ip
self.rest_username = serverInfo.rest_username
self.rest_password = serverInfo.rest_password
self.username = serverInfo.es_username
self.password = serverInfo.es_password
self.port = 9091 # serverInfo.port
self.baseUrl = "http://{0}:{1}/".format(self.ip, self.port)
self.capiBaseUrl = self.baseUrl
self.esHttpUrl = "http://{0}:9200".format(self.ip)
self.http_port = str(int(self.port) + 109)
self.proto = proto
self.conn = ES(server=self.esHttpUrl)
self.manager = managers.Cluster(self.conn)
self.test_params = TestInputSingleton.input
self.docs = None
def try_conn(self):
"""Try a new connection to the Elasticsearch."""
try:
self.conn = ES(self.conn_strs, timeout=5)
self.connected = True
except NoServerAvailable:
print("Error connecting to elasticsearch for logging")
def init():
conn = ES('127.0.0.1:9200')
try:
conn.delete_index("zhihu")
except:
pass
conn.create_index("zhihu")
mapping = {
u'id': {
'store': 'yes',
'type': u'integer'
},
u'link': {
'store': 'yes',
'type': u'string'
},
u'title': {
'boost': 1.0,
'index': 'analyzed',
'store': 'yes',
'type': u'string'
},
}
conn.put_mapping("answer", {'properties': mapping}, ["zhihu"])
for item in Data().getData():
conn.index(item, "zhihu", "answer", item['id'])
conn.refresh(["zhihu"])
return redirect('/list')
def get_related_collections(collection):
related_collections = []
conn = ES(['127.0.0.1:9200'])
conn.default_indices = FACET_INDEX
conn.refresh(FACET_INDEX)
q ={"query": {
"bool" : {
"must_not" : {"term" : { "uid" : collection.uid }},
"should" : [
{"terms" : { "subject" : [collection.subject] }},
{"terms" : { "topic" : [collection.topic] }},
],
"minimum_should_match" : 1,
}
}
}
try :
query = json.dumps(q)
url = "http://localhost:9200/%s/_search" % FACET_INDEX
response = urllib2.urlopen(url, query)
result = json.loads(response.read())
for res in result['hits']['hits']:
related_collections.append(res['_source'])
except Exception:
pass
return related_collections
def multi_param_search(request):
log_results = None
es = ES() # create elastic seach object
if request.method == 'POST': # if the search form is submitted
filters_list = []
# loop on each search param and check if it has value to add it to filter list
for param in [
"version", "ip_header_length", "ttl", "protocol",
"source_address", "destination_address", "source_port",
"dest_port", "sequence_number", "acknowledgement",
"tcp_header_length", "data", "datetime"
]:
if request.POST.get(param) != '':
q_param = TermFilter(param, request.POST.get(param))
filters_list.append(q_param)
if len(filters_list
) != 0: # if there is filter params get the results
orq = ANDFilter(filters_list)
q = FilteredQuery(MatchAllQuery(), orq)
log_results = es.search(q, indices=index_name, doc_types=type_name)
else:
log_results = None
elif request.method == 'GET': # get all packet when get the search page
log_results = es.search(MatchAllQuery(),
indices=index_name,
doc_types=type_name)
return render(request, 'multi_param_search.html',
{'log_results': log_results})
def get_related_videos(video):
related_videos = []
conn = ES(['127.0.0.1:9200'])
conn.default_indices = VIDEO_INDEX
conn.refresh(VIDEO_INDEX)
q = {
"query": {
"bool": {
"should": [
{"term" : { "uid" : video.uid } },
{"terms" : { "category" : [video.category]}},
{"terms" : { "topic" : [video.topic]}},
{"terms" : { "language" : [video.language]}}
],
"minimum_should_match" : 1
}
}
}
try:
query = json.dumps(q)
url = "http://localhost:9200/%s/_search" % VIDEO_INDEX
response = urllib2.urlopen(url, query)
result = json.loads(response.read())
for res in result['hits']['hits']:
related_videos.append(res['_source'])
except Exception:
pass
return related_videos
def conn(self):
if self.tdata.conn is None:
self.tdata.conn = ES(self.registry.connection_string,
bulk_size=self.bulk_size,
max_retries=self.max_retries,
timeout=self.timeout)
return self.tdata.conn
def main(argv):
start = 1
if len(sys.argv) > 1:
if sys.argv[1]:
start = sys.argv[1]
bulksize = 1000
es = ES(("http", "localhost", 9200), bulk_size=bulksize)
c0 = 0
t0 = time.time()
c1 = 0
t1 = time.time()
for n in range(start, start + 1000000):
result = es.index(
{
'a': random_string_generator(),
'b': random_string_generator(),
'c': random_string_generator(),
'd': random_string_generator(),
'e': random_string_generator(),
'f': random_string_generator(),
'g': random_string_generator(),
'h': random_string_generator(),
'i': random_string_generator(),
'j': random_string_generator(),
'k': random_string_generator(),
'l': random_string_generator(),
'm': random_string_generator(),
'n': random_string_generator(),
'o': random_string_generator(),
'p': random_string_generator(),
'q': random_string_generator(),
'r': random_string_generator(),
's': random_string_generator(),
't': random_string_generator(),
'u': random_string_generator(),
'v': random_string_generator(),
'w': random_string_generator(),
'x': random_string_generator(),
'y': random_string_generator(),
'z': random_string_generator()
},
'pyindex',
'pytype',
n,
bulk=True)
c0 = c0 + bulksize
c1 = c1 + bulksize
if result:
d0 = (time.time() - t0)
d1 = (time.time() - t1)
now = datetime.datetime.utcnow()
print("{0},{1},{2},{3},{4},{5},{6},{7}".format(
now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), result.took, c0, d0,
c0 / (d0 * bulksize), c1, d1, c1 / (d1 * bulksize)))
c1 = 0
t1 = time.time()
def __init__(self,
data_type,
csv_path="/tmp/",
es_hosts=("http://localhost:9200", )):
self.data_type = data_type
self.doc_type = "ticketnetwork_%s" % self.data_type
self.csv_path = csv_path
self.es = ES(es_hosts)
def __init__(self):
from pyes import ES
self.settings = get_project_settings()
if self.settings['ELASTICSEARCH_PORT']:
uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT'])
else:
uri = "%s" % (self.settings['ELASTICSEARCH_SERVER'])
self.es = ES([uri])
def es_deindex(self):
conn = ES(settings.ES_SERVERS, basic_auth=settings.ES_AUTH)
try:
conn.delete(index=self.tenant.slug,
doc_type=self.Meta.document_type,
id=meta.id)
except:
pass
def connect(self, connection_pool=1, bulk_size=10):
update_connection_pool(connection_pool)
try:
self.connection = ES(self.servers, bulk_size=bulk_size)
except NoServerAvailable:
self._log.error('Failed to connect to elastic search server')
return False
return True
def tearDown(self):
self.log.warning("before tearDown es")
self._unlink_es_cluster()
self._stop_es_replication()
if self.es_host != None:
conn = ES(self.es_host + ":9200")
conn.delete_index_if_exists("default")
super(ElasticSearchSupport, self).tearDown()
self.log.warning("after tearDown es")
def __init__(self, *args, **kwargs):
self._dirty = set()
# We have to wait for the elastic container to start or things go
# sideways.
# TODO: Check status properly somehow (straight HTTP request, perhaps)
time.sleep(30)
self._elastic = ES(ELASTIC_URL, max_retries=100)
self._finalize = Finalize(self, self.sync, exitpriority=5)
super(ControlPlaneScheduler, self).__init__(*args, **kwargs)
def __init__(self, settings):
basic_auth = {'username': settings.get('ELASTICSEARCH_USERNAME'),
'password': settings.get('ELASTICSEARCH_PASSWORD')}
if settings.get('ELASTICSEARCH_PORT'):
uri = "%s:%d" % (settings.get('ELASTICSEARCH_SERVER'), settings.get('ELASTICSEARCH_PORT'))
else:
uri = "%s" % (settings.get('ELASTICSEARCH_SERVER'))
self.es = ES([uri], basic_auth=basic_auth)
self.settings = settings
def find_BID_in_SBN(bid, es_server="localhost:9200"):
sbn_bid = to_iccu_bid(bid)
q = TermQuery('codiceIdentificativo', sbn_bid)
es_conn = ES(server=es_server)
resultset = list(es_conn.search(query=q, indices="iccu"))
if (len(resultset) > 0):
return resultset
else:
return None
def get_es_conn(es_url, index, alias=None):
"""Create connection and create index if it doesn't exist."""
conn = ES(es_url)
if not conn.indices.exists_index(index):
conn.indices.create_index(index)
if alias is not None:
conn.indices.add_alias(alias, [index])
return conn
def searchCompletions(request):
searchString = request.GET.get('searchString')
maxCount = int(request.GET.get('maxCount'))
conn = ES(['127.0.0.1:9200'])
conn.default_indices = COMPLETION_INDEX
conn.refresh(COMPLETION_INDEX)
q = {
"query": {
"query_string": {
"fields": ["searchTerm.partial"],
"query": searchString
}
},
"facets": {
"facet": {
"terms": {
"fields": ["searchTerm"],
"size": MAX_RESULT_SIZE
}
}
},
"size": maxCount
}
try:
query = json.dumps(q)
url = "http://localhost:9200/%s/_search" % COMPLETION_INDEX
response = urllib2.urlopen(url, query)
result = json.loads(response.read())
result_list = []
done_list = []
for res in result['hits']['hits']:
if res['_source']['type'] != "Collections":
result_list.append(res['_source'])
res['_source']['count'] = 0
elif res['_source']['searchTerm'] not in done_list:
val = str(res['_source']['searchTerm']).lower()
for term in result['facets']['facet']['terms']:
if val == term['term']:
res['_source']['count'] = term['count']
done_list.append(res['_source']['searchTerm'])
result_list.append(res['_source'])
if len(result_list) == 0:
result_list.append(
{"searchTerm": "No Results"}
) # for now just displaying no results when nothing is found in completion
resp = json.dumps({
"responseCode": "OK",
"requestParameters": {
"searchString": searchString,
"maxCount": unicode(maxCount)
},
"completions": result_list,
"totalCount": unicode(maxCount)
})
return HttpResponse(resp)
except Exception, ex:
return HttpResponse('0')
def set_connection(self, project):
logger.debug('Setting up connection')
if self.es_conn is None:
try:
cs = self.get_option('es_conn_string', project)
logger.debug('Creating connection to %s', cs)
self.es_conn = ES(cs)
except Exception, e:
logger.warning('Error setting up the connection: %s', e)
return
def __init__(self, name):
log = open(name, "wb")
self.log = log
self.conn = ES(("http", "127.0.0.1", 9200),
timeout=300.0,
log_curl=True,
dump_curl=log)
self.index_name = "test-index"
self.document_type = "test-type"
self.conn.delete_index_if_exists(self.index_name)
self.init_default_index()
def __init__(self):
self.settings = get_project_settings()
basic_auth = {'username': self.settings['ELASTICSEARCH_USERNAME'], 'password': self.settings['ELASTICSEARCH_PASSWORD']}
if self.settings['ELASTICSEARCH_PORT']:
uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT'])
else:
uri = "%s" % (self.settings['ELASTICSEARCH_SERVER'])
self.es = ES([uri], basic_auth=basic_auth)
def __init__(self, url, auto_commit=True, unique_key='_id'):
"""Verify Elastic URL and establish a connection.
"""
if verify_url(url) is False:
raise SystemError
self.elastic = ES(server=url)
self.auto_commit = auto_commit
self.doc_type = 'string' # default type is string, change if needed
self.unique_key = unique_key
if auto_commit:
self.run_auto_commit()
def processData(esurl, esindex, estype, shpPath, simplify, tolerance,
startfrom):
# Open a file for reading
try:
with open(shpPath):
pass
except IOError:
print 'Unable to locate file: ' + shpPath
#open the es connection
from pyes import ES
conn = ES(esurl, timeout=60, bulk_size=10)
#check that a tolerance is passed when simplifying.
if (simplify == True):
if (tolerance == None):
raise ValueError(
'You must pass a valid tolerance if simplifying geometry')
#use fiona to open the shapefile and read it
try:
with fiona.open(shpPath) as source:
for f in source:
featid = int(f['id'])
if (featid > startfrom):
#grab the geom
from shapely.geometry import shape
geom = shape(f['geometry'])
#simplify if required
if (validateGeometry(geom)):
if (simplify == True):
geom = simplifyGeometry(geom, tolerance)
#if the geom is valid then push it into es
if (validateGeometry(geom)):
data = json.dumps(f)
key = f['id']
conn.index(data, esindex, estype, key, bulk=True)
else:
logging.error('Invalid Geometry: ' + f['id'])
except:
raise
def get_es():
"""Return one es object."""
if not hasattr(_local, 'es'):
timeout = getattr(settings, 'ES_TIMEOUT', 1)
dump = getattr(settings, 'ES_DUMP_CURL', False)
if (not thrift_enable
and not settings.ES_HOSTS[0].split(':')[1].startswith('92')):
raise ValueError('ES_HOSTS is not set to a valid port starting '
'with 9200-9299 range. Other ports are valid '
'if using pythrift.')
_local.es = ES(settings.ES_HOSTS,
default_indexes=[settings.ES_INDEXES['default']],
timeout=timeout,
dump_curl=dump)
return _local.es
def setUp(self):
self.es_host = None
self.es_cluster_name = None
self._state = []
super(ElasticSearchSupport, self).setUp()
self.es_host = self.input.param("es_host", "127.0.0.1")
self.es_port = self.input.param("es_port", 9091)
conn = ES(self.es_host + ":9200")
if not self.input.param("skip_cleanup", True) or self.case_number == 1:
conn.delete_index_if_exists("default")
conn.create_index("default")
self.log.warning("waiting for ES index to be ready to use")
time.sleep(30)
self._link_es_cluster()
self._start_es_replication()
self.log.warning("after setUp es")
def get_es_connection(cls, **kwargs):
"""
Return a PYES connection object that can be reused by other models
"""
# TODO: Raise an exception if the configuration object is not
# created ?
# Or create one on the fly when connection is requested ?
configuration = cls(1)
logger = cls.get_logger()
if not configuration.settings_updated:
logger.warning('Settings are not updated on index')
return ES(configuration.servers.split(','),
default_indices=[configuration.index_name],
**kwargs)
def setup_elastic_search():
facet_index = dg.settings.FACET_INDEX
completion_index = dg.settings.COMPLETION_INDEX
video_index = dg.settings.VIDEO_INDEX
conn = ES(['127.0.0.1:9200'])
# FACET SEARCH
custom_create_index(conn, facet_index, SETTINGS, FACET_MAPPING)
enter_data_into_facet_search(conn, facet_index)
# COMPLETION SEARCH
custom_create_index(conn, completion_index, SETTINGS, COMPLETION_MAPPING)
enter_data_into_completion_search(conn, completion_index)
# VIDEO SEARCH
custom_create_index(conn, video_index, SETTINGS, VIDEO_MAPPING)
enter_data_into_video_search(conn, video_index)
def __init__(self):
'''
Constructor
'''
param = {
'default_ini': '%s%s' % (settings.USERAPP_DIR, 'userconfig.ini'),
'default_value_map': {}
}
self.indexcfg = Config.config(**param)
self.endpoint = self.indexcfg.get_config(
'indexing', 'pyes_endpoint')
self.indexing_tags = [e.strip() for e in self.indexcfg.get_config(
'indexing', 'index').split(',')]
self._conn = ES(self.endpoint)
self._mapper = ESMapper(self.indexcfg)
# create the indexes
self.create_indexes(self.indexing_tags)
def single_param_search(request):
log_results = None
es = ES() # create elastic seach object
if request.method == 'POST': # if the search form is submitted
# filter with search param and search tearm
q1 = TermFilter(request.POST.get('searchby'),
request.POST.get('searchterm'))
orq = ORFilter([q1])
q = FilteredQuery(MatchAllQuery(), orq)
log_results = es.search(
q, indices=index_name,
doc_types=type_name) # get the filtered data from elasticsearch
elif request.method == 'GET': # get all packet when get the search page
log_results = es.search(MatchAllQuery(),
indices=index_name,
doc_types=type_name)
return render(request, 'single_param_search.html',
{'log_results': log_results})
