def get_es_conn(es_url, index): """Create connection and create index if it doesn't exist.""" conn = ES(es_url) if not conn.indices.exists_index(index): conn.indices.create_index(index) return conn
def __init__(self, serverInfo, proto="http"): #serverInfo can be a json object #only connect pyes to master es node #in the case that other nodes are taken down #because http requests will fail # TODO: dynamic master node detection if isinstance(serverInfo, dict): self.ip = serverInfo["ip"] self.rest_username = serverInfo["username"] self.rest_password = serverInfo["password"] self.username = serverInfo["es_username"] self.password = serverInfo["es_password"] self.port = 9091 #serverInfo["port"] else: self.ip = serverInfo.ip self.rest_username = serverInfo.rest_username self.rest_password = serverInfo.rest_password self.username = serverInfo.es_username self.password = serverInfo.es_password self.port = 9091 # serverInfo.port self.baseUrl = "http://{0}:{1}/".format(self.ip, self.port) self.capiBaseUrl = self.baseUrl self.esHttpUrl = "http://{0}:9200".format(self.ip) self.http_port = str(int(self.port) + 109) self.proto = proto self.conn = ES(server=self.esHttpUrl) self.manager = managers.Cluster(self.conn) self.test_params = TestInputSingleton.input self.docs = None
def try_conn(self): """Try a new connection to the Elasticsearch.""" try: self.conn = ES(self.conn_strs, timeout=5) self.connected = True except NoServerAvailable: print("Error connecting to elasticsearch for logging")
def init(): conn = ES('127.0.0.1:9200') try: conn.delete_index("zhihu") except: pass conn.create_index("zhihu") mapping = { u'id': { 'store': 'yes', 'type': u'integer' }, u'link': { 'store': 'yes', 'type': u'string' }, u'title': { 'boost': 1.0, 'index': 'analyzed', 'store': 'yes', 'type': u'string' }, } conn.put_mapping("answer", {'properties': mapping}, ["zhihu"]) for item in Data().getData(): conn.index(item, "zhihu", "answer", item['id']) conn.refresh(["zhihu"]) return redirect('/list')
def get_related_collections(collection): related_collections = [] conn = ES(['127.0.0.1:9200']) conn.default_indices = FACET_INDEX conn.refresh(FACET_INDEX) q ={"query": { "bool" : { "must_not" : {"term" : { "uid" : collection.uid }}, "should" : [ {"terms" : { "subject" : [collection.subject] }}, {"terms" : { "topic" : [collection.topic] }}, ], "minimum_should_match" : 1, } } } try : query = json.dumps(q) url = "http://localhost:9200/%s/_search" % FACET_INDEX response = urllib2.urlopen(url, query) result = json.loads(response.read()) for res in result['hits']['hits']: related_collections.append(res['_source']) except Exception: pass return related_collections
def multi_param_search(request): log_results = None es = ES() # create elastic seach object if request.method == 'POST': # if the search form is submitted filters_list = [] # loop on each search param and check if it has value to add it to filter list for param in [ "version", "ip_header_length", "ttl", "protocol", "source_address", "destination_address", "source_port", "dest_port", "sequence_number", "acknowledgement", "tcp_header_length", "data", "datetime" ]: if request.POST.get(param) != '': q_param = TermFilter(param, request.POST.get(param)) filters_list.append(q_param) if len(filters_list ) != 0: # if there is filter params get the results orq = ANDFilter(filters_list) q = FilteredQuery(MatchAllQuery(), orq) log_results = es.search(q, indices=index_name, doc_types=type_name) else: log_results = None elif request.method == 'GET': # get all packet when get the search page log_results = es.search(MatchAllQuery(), indices=index_name, doc_types=type_name) return render(request, 'multi_param_search.html', {'log_results': log_results})
def get_related_videos(video): related_videos = [] conn = ES(['127.0.0.1:9200']) conn.default_indices = VIDEO_INDEX conn.refresh(VIDEO_INDEX) q = { "query": { "bool": { "should": [ {"term" : { "uid" : video.uid } }, {"terms" : { "category" : [video.category]}}, {"terms" : { "topic" : [video.topic]}}, {"terms" : { "language" : [video.language]}} ], "minimum_should_match" : 1 } } } try: query = json.dumps(q) url = "http://localhost:9200/%s/_search" % VIDEO_INDEX response = urllib2.urlopen(url, query) result = json.loads(response.read()) for res in result['hits']['hits']: related_videos.append(res['_source']) except Exception: pass return related_videos
def conn(self): if self.tdata.conn is None: self.tdata.conn = ES(self.registry.connection_string, bulk_size=self.bulk_size, max_retries=self.max_retries, timeout=self.timeout) return self.tdata.conn
def main(argv): start = 1 if len(sys.argv) > 1: if sys.argv[1]: start = sys.argv[1] bulksize = 1000 es = ES(("http", "localhost", 9200), bulk_size=bulksize) c0 = 0 t0 = time.time() c1 = 0 t1 = time.time() for n in range(start, start + 1000000): result = es.index( { 'a': random_string_generator(), 'b': random_string_generator(), 'c': random_string_generator(), 'd': random_string_generator(), 'e': random_string_generator(), 'f': random_string_generator(), 'g': random_string_generator(), 'h': random_string_generator(), 'i': random_string_generator(), 'j': random_string_generator(), 'k': random_string_generator(), 'l': random_string_generator(), 'm': random_string_generator(), 'n': random_string_generator(), 'o': random_string_generator(), 'p': random_string_generator(), 'q': random_string_generator(), 'r': random_string_generator(), 's': random_string_generator(), 't': random_string_generator(), 'u': random_string_generator(), 'v': random_string_generator(), 'w': random_string_generator(), 'x': random_string_generator(), 'y': random_string_generator(), 'z': random_string_generator() }, 'pyindex', 'pytype', n, bulk=True) c0 = c0 + bulksize c1 = c1 + bulksize if result: d0 = (time.time() - t0) d1 = (time.time() - t1) now = datetime.datetime.utcnow() print("{0},{1},{2},{3},{4},{5},{6},{7}".format( now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"), result.took, c0, d0, c0 / (d0 * bulksize), c1, d1, c1 / (d1 * bulksize))) c1 = 0 t1 = time.time()
def __init__(self, data_type, csv_path="/tmp/", es_hosts=("http://localhost:9200", )): self.data_type = data_type self.doc_type = "ticketnetwork_%s" % self.data_type self.csv_path = csv_path self.es = ES(es_hosts)
def __init__(self): from pyes import ES self.settings = get_project_settings() if self.settings['ELASTICSEARCH_PORT']: uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT']) else: uri = "%s" % (self.settings['ELASTICSEARCH_SERVER']) self.es = ES([uri])
def es_deindex(self): conn = ES(settings.ES_SERVERS, basic_auth=settings.ES_AUTH) try: conn.delete(index=self.tenant.slug, doc_type=self.Meta.document_type, id=meta.id) except: pass
def connect(self, connection_pool=1, bulk_size=10): update_connection_pool(connection_pool) try: self.connection = ES(self.servers, bulk_size=bulk_size) except NoServerAvailable: self._log.error('Failed to connect to elastic search server') return False return True
def tearDown(self): self.log.warning("before tearDown es") self._unlink_es_cluster() self._stop_es_replication() if self.es_host != None: conn = ES(self.es_host + ":9200") conn.delete_index_if_exists("default") super(ElasticSearchSupport, self).tearDown() self.log.warning("after tearDown es")
def __init__(self, *args, **kwargs): self._dirty = set() # We have to wait for the elastic container to start or things go # sideways. # TODO: Check status properly somehow (straight HTTP request, perhaps) time.sleep(30) self._elastic = ES(ELASTIC_URL, max_retries=100) self._finalize = Finalize(self, self.sync, exitpriority=5) super(ControlPlaneScheduler, self).__init__(*args, **kwargs)
def __init__(self, settings): basic_auth = {'username': settings.get('ELASTICSEARCH_USERNAME'), 'password': settings.get('ELASTICSEARCH_PASSWORD')} if settings.get('ELASTICSEARCH_PORT'): uri = "%s:%d" % (settings.get('ELASTICSEARCH_SERVER'), settings.get('ELASTICSEARCH_PORT')) else: uri = "%s" % (settings.get('ELASTICSEARCH_SERVER')) self.es = ES([uri], basic_auth=basic_auth) self.settings = settings
def find_BID_in_SBN(bid, es_server="localhost:9200"): sbn_bid = to_iccu_bid(bid) q = TermQuery('codiceIdentificativo', sbn_bid) es_conn = ES(server=es_server) resultset = list(es_conn.search(query=q, indices="iccu")) if (len(resultset) > 0): return resultset else: return None
def get_es_conn(es_url, index, alias=None): """Create connection and create index if it doesn't exist.""" conn = ES(es_url) if not conn.indices.exists_index(index): conn.indices.create_index(index) if alias is not None: conn.indices.add_alias(alias, [index]) return conn
def searchCompletions(request): searchString = request.GET.get('searchString') maxCount = int(request.GET.get('maxCount')) conn = ES(['127.0.0.1:9200']) conn.default_indices = COMPLETION_INDEX conn.refresh(COMPLETION_INDEX) q = { "query": { "query_string": { "fields": ["searchTerm.partial"], "query": searchString } }, "facets": { "facet": { "terms": { "fields": ["searchTerm"], "size": MAX_RESULT_SIZE } } }, "size": maxCount } try: query = json.dumps(q) url = "http://localhost:9200/%s/_search" % COMPLETION_INDEX response = urllib2.urlopen(url, query) result = json.loads(response.read()) result_list = [] done_list = [] for res in result['hits']['hits']: if res['_source']['type'] != "Collections": result_list.append(res['_source']) res['_source']['count'] = 0 elif res['_source']['searchTerm'] not in done_list: val = str(res['_source']['searchTerm']).lower() for term in result['facets']['facet']['terms']: if val == term['term']: res['_source']['count'] = term['count'] done_list.append(res['_source']['searchTerm']) result_list.append(res['_source']) if len(result_list) == 0: result_list.append( {"searchTerm": "No Results"} ) # for now just displaying no results when nothing is found in completion resp = json.dumps({ "responseCode": "OK", "requestParameters": { "searchString": searchString, "maxCount": unicode(maxCount) }, "completions": result_list, "totalCount": unicode(maxCount) }) return HttpResponse(resp) except Exception, ex: return HttpResponse('0')
def set_connection(self, project): logger.debug('Setting up connection') if self.es_conn is None: try: cs = self.get_option('es_conn_string', project) logger.debug('Creating connection to %s', cs) self.es_conn = ES(cs) except Exception, e: logger.warning('Error setting up the connection: %s', e) return
def __init__(self, name): log = open(name, "wb") self.log = log self.conn = ES(("http", "127.0.0.1", 9200), timeout=300.0, log_curl=True, dump_curl=log) self.index_name = "test-index" self.document_type = "test-type" self.conn.delete_index_if_exists(self.index_name) self.init_default_index()
def __init__(self): self.settings = get_project_settings() basic_auth = {'username': self.settings['ELASTICSEARCH_USERNAME'], 'password': self.settings['ELASTICSEARCH_PASSWORD']} if self.settings['ELASTICSEARCH_PORT']: uri = "%s:%d" % (self.settings['ELASTICSEARCH_SERVER'], self.settings['ELASTICSEARCH_PORT']) else: uri = "%s" % (self.settings['ELASTICSEARCH_SERVER']) self.es = ES([uri], basic_auth=basic_auth)
def __init__(self, url, auto_commit=True, unique_key='_id'): """Verify Elastic URL and establish a connection. """ if verify_url(url) is False: raise SystemError self.elastic = ES(server=url) self.auto_commit = auto_commit self.doc_type = 'string' # default type is string, change if needed self.unique_key = unique_key if auto_commit: self.run_auto_commit()
def processData(esurl, esindex, estype, shpPath, simplify, tolerance, startfrom): # Open a file for reading try: with open(shpPath): pass except IOError: print 'Unable to locate file: ' + shpPath #open the es connection from pyes import ES conn = ES(esurl, timeout=60, bulk_size=10) #check that a tolerance is passed when simplifying. if (simplify == True): if (tolerance == None): raise ValueError( 'You must pass a valid tolerance if simplifying geometry') #use fiona to open the shapefile and read it try: with fiona.open(shpPath) as source: for f in source: featid = int(f['id']) if (featid > startfrom): #grab the geom from shapely.geometry import shape geom = shape(f['geometry']) #simplify if required if (validateGeometry(geom)): if (simplify == True): geom = simplifyGeometry(geom, tolerance) #if the geom is valid then push it into es if (validateGeometry(geom)): data = json.dumps(f) key = f['id'] conn.index(data, esindex, estype, key, bulk=True) else: logging.error('Invalid Geometry: ' + f['id']) except: raise
def get_es(): """Return one es object.""" if not hasattr(_local, 'es'): timeout = getattr(settings, 'ES_TIMEOUT', 1) dump = getattr(settings, 'ES_DUMP_CURL', False) if (not thrift_enable and not settings.ES_HOSTS[0].split(':')[1].startswith('92')): raise ValueError('ES_HOSTS is not set to a valid port starting ' 'with 9200-9299 range. Other ports are valid ' 'if using pythrift.') _local.es = ES(settings.ES_HOSTS, default_indexes=[settings.ES_INDEXES['default']], timeout=timeout, dump_curl=dump) return _local.es
def setUp(self): self.es_host = None self.es_cluster_name = None self._state = [] super(ElasticSearchSupport, self).setUp() self.es_host = self.input.param("es_host", "127.0.0.1") self.es_port = self.input.param("es_port", 9091) conn = ES(self.es_host + ":9200") if not self.input.param("skip_cleanup", True) or self.case_number == 1: conn.delete_index_if_exists("default") conn.create_index("default") self.log.warning("waiting for ES index to be ready to use") time.sleep(30) self._link_es_cluster() self._start_es_replication() self.log.warning("after setUp es")
def get_es_connection(cls, **kwargs): """ Return a PYES connection object that can be reused by other models """ # TODO: Raise an exception if the configuration object is not # created ? # Or create one on the fly when connection is requested ? configuration = cls(1) logger = cls.get_logger() if not configuration.settings_updated: logger.warning('Settings are not updated on index') return ES(configuration.servers.split(','), default_indices=[configuration.index_name], **kwargs)
def setup_elastic_search(): facet_index = dg.settings.FACET_INDEX completion_index = dg.settings.COMPLETION_INDEX video_index = dg.settings.VIDEO_INDEX conn = ES(['127.0.0.1:9200']) # FACET SEARCH custom_create_index(conn, facet_index, SETTINGS, FACET_MAPPING) enter_data_into_facet_search(conn, facet_index) # COMPLETION SEARCH custom_create_index(conn, completion_index, SETTINGS, COMPLETION_MAPPING) enter_data_into_completion_search(conn, completion_index) # VIDEO SEARCH custom_create_index(conn, video_index, SETTINGS, VIDEO_MAPPING) enter_data_into_video_search(conn, video_index)
def __init__(self): ''' Constructor ''' param = { 'default_ini': '%s%s' % (settings.USERAPP_DIR, 'userconfig.ini'), 'default_value_map': {} } self.indexcfg = Config.config(**param) self.endpoint = self.indexcfg.get_config( 'indexing', 'pyes_endpoint') self.indexing_tags = [e.strip() for e in self.indexcfg.get_config( 'indexing', 'index').split(',')] self._conn = ES(self.endpoint) self._mapper = ESMapper(self.indexcfg) # create the indexes self.create_indexes(self.indexing_tags)
def single_param_search(request): log_results = None es = ES() # create elastic seach object if request.method == 'POST': # if the search form is submitted # filter with search param and search tearm q1 = TermFilter(request.POST.get('searchby'), request.POST.get('searchterm')) orq = ORFilter([q1]) q = FilteredQuery(MatchAllQuery(), orq) log_results = es.search( q, indices=index_name, doc_types=type_name) # get the filtered data from elasticsearch elif request.method == 'GET': # get all packet when get the search page log_results = es.search(MatchAllQuery(), indices=index_name, doc_types=type_name) return render(request, 'single_param_search.html', {'log_results': log_results})