def main_method_to_loop(): global nodes_stats_before global nodes_stats_after get_node_names() # Get get Initial stats before sleep es = Elasticsearch(es_url, verify_certs=False, connection_class=RequestsHttpConnection) print("[INFO] [" + datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "]: Fetch Node BEFORE Stats\n") nodes_stats_before = es.nodes.stats() # Do Loop for 5 times with 2sec sleep and update the cluster stats for i in range(1, 5): print("[INFO] [" + datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "]: Fetch Cluster Stats") get_cluster_stats() time.sleep(2) # Get the after stats. This will be after some time print("[INFO] [" + datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "]: Fetch Node AFTER Stats\n") nodes_stats_after = es.nodes.stats() # Now using before and after node stats, Calculate ES Metrics measure_es_metrics()
def main(args: argparse.ArgumentParser(), logger: logging.getLogger()) -> list: """ Setups Client, Makes the call at Elasticsearch with the query and returns a sorted list of dicitonaries Parameters ---------- args: argparse.ArgumentParser The command line arguments given logger: logging.getLogger logger on DEBUG level Returns ------- ordered list of dictionaries {'questionId: ..., 'score: } """ # setup Client logger.info("# setup Client") client = Elasticsearch() # setup query logger.info("# setup query") query_request = setup_query(args) # make request logger.info("# make request") bm25_scores = make_request(query_request, client) return bm25_scores
def index_kb(all_docs_kb: dict, client: Elasticsearch(), args: argparse.ArgumentParser(), logger: logging.getLogger()): """ Indexes the KB Parameters ---------- args: argparse.ArgumentParser The command line arguments given logger: logging.getLogger logger on DEBUG level Returns ------- None """ # Delete Index client.indices.delete(index=INDEX_NAME, ignore=[404]) # Create Index with open(MAPPING_JSON_FILE) as mapping_json_file: source = mapping_json_file.read().strip() client.indices.create(index=INDEX_NAME, body=source) # upload the KB logger.info("# upload the KB") bulk(client, all_docs_kb) return
def __init__(self, hosts=None, index="metrics", doc_type="metric", index_pattern="{index}-{date:%Y.%m.%d}", *args, **kwargs): # Assign these in the backend as they are needed when writing metrics # to elasticsearch self.index = index self.doc_type = doc_type self.index_pattern = index_pattern # setup the client self.client = Elasticsearch(hosts=hosts, *args, **kwargs) # ensure the index is created try: self._setup_index() except TransportError as exc: logger.error('index setup error %r', exc) try: self._setup_mapping() except TransportError as exc: logger.error('mapping setup error %r', exc)
def make_request(query_request: dict, client: Elasticsearch()) -> list: """ Makes a Call to an ES-Client with the query_request and returns a sorted list of dicitonaries containing the questionId and Score Parameters ---------- query_request: dict dictionary containing the request to make at the client client: Elasticsearch() Elasticsearch client Returns ------- ordered list of dictionaries {'questionId: ..., 'score: } """ # make query request response = client.search( index=INDEX_NAME, # index names to search body=query_request) # filter out questionId and score hits = response["hits"]["hits"] bm25_scores = [{ 'questionId': hit["_source"]["questionId"], 'score': hit["_score"] } for hit in hits] return bm25_scores
def create_app(config_class=Config): app = Flask(__name__) app.config.from_object(config_class) db.init_app(app) migrate.init_app(app, db) login.init_app(app) mail.init_app(app) bootstrap.init_app(app) moment.init_app(app) babel.init_app(app) app.elasticsearch = Elasticsearch([app.config['ELASTICSEARCH_URL']]) \ if app.config['ELASTICSEARCH_URL'] else None from app.errors import bp as errors_bp app.register_blueprint(errors_bp) from app.auth import bp as auth_bp app.register_blueprint(auth_bp, url_prefix='/auth') from app.main import bp as main_bp app.register_blueprint(main_bp) if not app.debug and not app.testing: if app.config['MAIL_SERVER']: auth = None if app.config['MAIL_USERNAME'] or app.config['MAIL_PASSWORD']: auth = (app.config['MAIL_USERNAME'], app.config['MAIL_PASSWORD']) secure = None if app.config['MAIL_USE_TLS']: secure = () mail_handler = SMTPHandler( mailhost=(app.config['MAIL_SERVER'], app.config['MAIL_PORT']), fromaddr='no-reply@' + app.config['MAIL_SERVER'], toaddrs=app.config['ADMINS'], subject='Microblog Failure', credentials=auth, secure=secure) mail_handler.setLevel(logging.ERROR) app.logger.addHandler(mail_handler) if not os.path.exists('logs'): os.mkdir('logs') file_handler = RotatingFileHandler('logs/microblog.log', maxBytes=10240, backupCount=10) file_handler.setFormatter( logging.Formatter('%(asctime)s %(levelname)s: %(message)s ' '[in %(pathname)s:%(lineno)d]')) file_handler.setLevel(logging.INFO) app.logger.addHandler(file_handler) app.logger.setLevel(logging.INFO) app.logger.info('Microblog startup') return app
def test_chunk_sent_from_different_threads(self, _process_bulk_chunk): actions = ({'x': i} for i in range(100)) results = list( helpers.parallel_bulk(Elasticsearch(), actions, thread_count=10, chunk_size=2)) self.assertTrue(len(set([r[1] for r in results])) > 1)
def main(args: argparse.ArgumentParser(), logger: logging.getLogger()): """asdf und nochmal asdf""" # load_live_model ( model, scaler, all_docs_kb, data_kb_with_vectors, qid_to_class, class_to_qid, without_stopwords, num_of_sentences, ) = load_model(args.KB_id) # set args parameters args.without_stopwords = without_stopwords args.num_of_sentences = num_of_sentences # load use and bm25-elasticsearch logger.info("# load use and bm25-elasticsearch") embed = initialize_use_model() client = Elasticsearch() # execute main index_elasticsearch logger.info("# execute main index_elasticsearch") index_kb(all_docs_kb, client, args, logger) while True: query = input("enter your query or q: ") if query == "q": print("Exiting") break args.query = query start = time.time() use_response = make_query_use(data_kb_with_vectors, embed, args, logger) end = time.time() print(end - start) bm25_response = make_query_bm25(client, args) response_vector_level_1 = encode_responses(bm25_response, use_response, len(model.classes_), qid_to_class).reshape(1, -1) scaler.transform(response_vector_level_1) response_vector_level_2 = model.predict_log_proba(response_vector_level_1) response_vector_level_2 = np.argsort(response_vector_level_2)[::-1] response = [class_to_qid[pos] for pos in response_vector_level_2[0]] return
def __init__(self, ES): self._add = ES #创建es客户端 self.es = Elasticsearch( self._add, # 启动前嗅探es集群服务器 sniff_on_start=True, # es集群服务器结点连接异常时是否刷新es节点信息 sniff_on_connection_fail=True, # 每60秒刷新节点信息 sniffer_timeout=60)
def initialize_elasticsearch(): n = 0 #initialize counter "n" while n <= 10: try: es = Elasticsearch(ES_SERVER) print("Initializing Elasticsearch...") return es except elasticsearch6.exceptions.ConnectionTimeout as err: print(err) n += 1 continue raise Exception
def elastic_search(): wait_for(f"http://{config.es_host}:{config.es_port}/") es = Elasticsearch([{"host": config.es_host, "port": config.es_port}]) def search(body: dict) -> None: es.indices.flush() es.indices.refresh() return es.search(index="test", body=body) try: yield search finally: if es.indices.exists("test"): es.indices.delete("test")
def main(): parser = argparse.ArgumentParser( prog="es-rebalance", formatter_class=argparse.RawDescriptionHelpFormatter, description="""\ Disk-usage-based rebalancing tool for ElasticSearch. Swaps large shards with small shards to balance disk usage between hosts. Starts one set of swaps per run. Swaps will happen asynchronously after running this script. Multiple runs may be needed to fully balance. This script is rack aware; it will not swap shards so that a primary and replica for the same index are on the same rack. Example: es-rebalance -u localhost:9200 --box-type hot --iterations 50 """) parser.add_argument("-u", "--url", required=True, action="append", help="URL to cluster. Can be specified multiple times for redundancy.") parser.add_argument("-b", "--box-type", required=True, help="Box type of nodes to rebalance. One of 'warm' or 'hot'.") parser.add_argument("-i", "--iterations", type=int, default=10, help="Maximum number of shards to exchange.") parser.add_argument("-p", "--shard-percentage", type=float, default=90, help="Don't exchange shards whose sizes are within this percent of each other, to avoid swapping similar-sized shards.") parser.add_argument("-P", "--node-percentage", type=float, default=10, help="Don't exchange between nodes whose sizes are within this many percentage points of each other.") parser.add_argument("-v", "--verbose", action="store_true", help="Print debug logs.") parser.add_argument("--execute", action="store_true", help="Run the plan. If not specified, will be a dry run.") args = parser.parse_args() logging.basicConfig(level=logging.INFO) if args.verbose: LOG.setLevel(logging.DEBUG) es = Elasticsearch(args.url) plan = Plan(es, args.box_type, args.shard_percentage, args.node_percentage) for i in range(args.iterations): if not plan.plan_step(): LOG.warn("Could not move anything, stopping early after %d iteration(s)", i+1) break plan.exec(dry_run=not args.execute) if not args.execute: LOG.warn("Finished dry run. Use `--execute` to run for real.")
def es_reporter(): es = Elasticsearch([{"host": config.es_host, "port": config.es_port}]) if not es.ping(): logger.error( "Cannot connect to Elasticsearch server. Host: %s, Port: %s", config.es_host, config.es_port, ) def on_message(message: Message): logger.debug("From queue: %s, received: %s", config.amqp_queue, message.body) try: validate_payload(message.body) except ValueError as err: logger.error("%s Message: %s", err, message.body) message.reject() logger.warning("Rejected: %s", message.body) return try: es.index(index=config.es_index, body=message.body, doc_type=config.es_doc_type) except ElasticsearchException as err: logger.error("Cannot modify index %s. Error: %s", config.es_index, err) message.reject() logger.warning("Rejected: %s", message.body) return logger.debug( "To index: %s, type: %s, document: %s", config.es_index, config.es_doc_type, message.body, ) message.ack() logger.debug("Acked: %s", message.body) try: yield on_message finally: try: es.close() except AttributeError: pass
def __init__(self, host, port, index_param, field): self.field = field self.index = index_param self.es = Elasticsearch([host + ':' + str(port)]) if not self.es.indices.exists(index=index_param): self.es.indices.create(index=index_param, body={ "mappings": { "_doc": { "properties": { field: { "type": "geo_point" } } } } })
def query_data(index, dls): hosts = ['192.168.5.11', '192.168.5.12', '192.168.5.14'] es = Elasticsearch(hosts=hosts, port=9200) res = es.search(index=index, scroll='1m', timeout='3s', size=1000, body=dls) mdata = res.get("hits").get("hits") if not mdata: print('empty!') scroll_id = res["_scroll_id"] total = res["hits"]["total"] for i in range(int(total / 1000)): res_scroll = es.scroll(scroll_id=scroll_id, scroll='1m') mdata += res_scroll["hits"]["hits"] result = mdata return result
def get_test_client(nowait=False, **kwargs): # construct kwargs from the environment kw = {'timeout': 30} if 'TEST_ES_CONNECTION' in os.environ: from elasticsearch6 import connection kw['connection_class'] = getattr(connection, os.environ['TEST_ES_CONNECTION']) kw.update(kwargs) client = Elasticsearch([os.environ.get('TEST_ES_SERVER', {})], **kw) # wait for yellow status for _ in range(1 if nowait else 100): try: client.cluster.health(wait_for_status='yellow') return client except ConnectionError: time.sleep(.1) else: # timeout raise SkipTest("Elasticsearch failed to start.")
def __init__(self, app): Config.__instance = self self.app = app self.client = Elasticsearch( app.config['ELASTICSEARCH_CLUSTER_NODES'] )
from elasticsearch6 import Elasticsearch, helpers import json import pprint from io import StringIO # ES IP주소와 포트(9200)로 연결한다. es = Elasticsearch("localhost:9200") def insertData(): index = "seoul-metro-logs-2019" with open( '/home/ec2-user/nclab/elastic-demos/seoul-metro-logs/data/seoul-metro-2019.logs', 'r') as logfile: for line in logfile: line = StringIO(line) doc = json.load(line) es.index(index="seoul-metro-logs-2019", doc_type="_doc", body=doc) insertData()
def get_records(report_date, elasticsearch_url): query = { 'query': { 'bool': { 'filter': [ { 'range': { 'date': { 'gte': report_date.strftime('%Y-%m-%d'), 'lte': report_date.strftime('%Y-%m-%d'), }, }, }, { 'match': { 'action': 'REST.GET.OBJECT', }, }, { 'range': { 'size': { 'gte': 2097152, }, }, }, { 'bool': { 'minimum_should_match': 1, 'should': [ { 'match': { 'response': 200, }, }, { 'match': { 'response': 206, }, }, ], }, }, { 'bool': { 'minimum_should_match': 1, 'should': [ { 'match_phrase': { 'bucket': f'asf-ngap2-p-*', }, }, { 'match_phrase': { 'bucket': f'asf-ngap2w-p-*', }, }, ], }, }, ], 'must_not': [ { 'match_phrase_prefix': { 'user_agent': 'RAIN Egress App for userid=', }, }, { 'match_phrase_prefix': { 'user_agent': 'Egress App for userid=', }, }, ], }, }, } desired_fields = [ 'ip', 'object', 'response', 'volume', 'size', 'user_agent', 'userid', 'date', 'eventid' ] index = f'dls.{report_date.year}.*' es_client = Elasticsearch(elasticsearch_url) results = scan(es_client, query=query, index=index, doc_type='log', _source_includes=desired_fields) records = (r['_source'] for r in results) return records
#!/usr/bin/env python from elasticsearch6 import Elasticsearch from socket import gethostname from time import sleep import json import pprint import salt.client whoami = gethostname() alert = salt.client.Caller() pp = pprint.PrettyPrinter(indent=4) es = Elasticsearch([{'host':'localhost','port':9200}]) S = [ "device", "rule_name", "action", "application", "severity", "threat_type", "rule_name", "rule_name", "src_ip", "dst_ip", "dst_port" ] Q = { "query": { "bool": { "must": [ { "terms": {"action": ["deny", "drop", "reset-client", "reset-server", "reset-both", "block-url", "block-ip", "random-drop", "sinkhole", "block"]} }, { "range": { "@timestamp": { "gte": "now-1h", "lte": "now" } }} ] } } }
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html #AUTOTHROTTLE_ENABLED = True # The initial download delay #AUTOTHROTTLE_START_DELAY = 5 # The maximum download delay to be set in case of high latencies #AUTOTHROTTLE_MAX_DELAY = 60 # The average number of requests Scrapy should be sending in parallel to # each remote server #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 # Enable showing throttling stats for every response received: #AUTOTHROTTLE_DEBUG = False # Enable and configure HTTP caching (disabled by default) # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings #HTTPCACHE_ENABLED = True #HTTPCACHE_EXPIRATION_SECS = 0 #HTTPCACHE_DIR = 'httpcache' #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' from elasticsearch6 import Elasticsearch es_host = Elasticsearch( ["192.168.0.253:9200", "192.168.0.252:9200"], #开启嗅探 sniff_on_start=True, # 节点没有响应时,进行刷新,重新连接 sniff_on_connection_fail=True, # 每 60 秒刷新一次 sniffer_timeout=60, #嗅探请求10s sniff_timeout=10)
from app import celery, create_app from elasticsearch6 import Elasticsearch app = create_app() es = Elasticsearch( [app.config.get('ES_HOST')], http_auth=(app.config.get('ES_USERNAME'), app.config.get('ES_PASSWORD')), port=app.config.get('ES_PORT'), ) app.app_context().push()
def get_client(): return Elasticsearch(hosts=[{ 'host': settings.ES_HOST, 'port': settings.ES_PORT }])
def main(): a_time = time.time() args = get_args() config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "config.ini") #config_file = "/sfs-grand-med-research/home/suyanan/scripts/SV/dbSV/dbsv-es-demos/config.ini" #input_file = os.path.join("/sfs-grand-med-research/home/suyanan/research/elasticsearch/demo_data","DBM19A3676-1.af_0.3.minre_0.3.maxre_2.0.filter.bed") #19578SV(814) #input_file = os.path.join("/sfs-grand-med-research/home/suyanan/research/elasticsearch/demo_data","demo.bed") #input_file = os.path.join("/sfs-grand-med-research/home/suyanan/research/elasticsearch/demo_data","demo.tra.bed") #input_file = os.path.join("/sfs-grand-med-research/home/suyanan/research/elasticsearch/demo_data","DBM19A3676-1.re2.filter.bed") #40746SV(7138TRA) input_file = args.input ##config paras config = configparser.ConfigParser() config.read(config_file) #es info es = config['es'] es_address = es['address'] client = Elasticsearch(es_address) #index mysv es_mysv = config['es-mysv'] es_mysv_index = es_mysv['index'] es_mysv_doc_type = es_mysv['doc_type'] MYSV_SIZE = int(es_mysv["SIZE"]) #index mysample es_sample = config['es-sample'] es_sample_index = es_sample['index'] es_sample_doc_type = es_sample['doc_type'] SAMPLE_SIZE = es_sample['SIZE'] #constant vars filter = config['filter'] MIN_RE = int(filter['MIN_RE']) BREAKPOINT = int(filter['BREAKPOINT']) OVERLAP = float(filter['OVERLAP']) SVTYPE_REGION_LIST = filter['SVTYPE_REGION_LIST'] nonStandardSVtype_list = filter['nonStandardSVtype'] #bed culomn of query svs query_dict = config['query'] #chrom_col,start_col,info_col ##dbSV样本个数和健康人样本个数的查询,一次性的 sample_num, samples_list, healthy_db_num, healthy_db_list = get_sample_data( client, es_sample_index, es_sample_doc_type, SAMPLE_SIZE) print("there are {} samples in dbsv, including {} healthy samples.".format( sample_num, healthy_db_num)) ##SV batch query from ES dbSV target_svs_list = get_batch_svs(input_file, query_dict) #stat_file = os.path.splitext(input_file)[0]+".dbsv.txt" stat_file = os.path.join( args.outdir, os.path.basename(input_file).strip("bed") + "dbsv.txt") with open(stat_file, "w") as out_io: out_list = [] for target_sv in target_svs_list: starter = time.time() target_chrom = target_sv["target_chrom"] target_start = target_sv["target_start"] target_svid = target_sv["target_svid"] target_end = target_sv["target_end"] target_svtype = target_sv[ "target_svtype"] #5svs and nonStandardSVtype_list target_svlen = target_sv["target_svlen"] target_chr2 = target_sv["target_chr2"] ##1st query and 2nd filter, filter_sample_list is sample:svid filter_num, filter_sample_list = get_sv_freq( client, es_mysv_index, es_mysv_doc_type, MIN_RE, BREAKPOINT, OVERLAP, SVTYPE_REGION_LIST, MYSV_SIZE, target_chrom, target_chr2, target_svtype, target_start, target_end, target_svlen, nonStandardSVtype_list) ##频率:基于dbSV所有样本 dbsv_freq = filter_num / sample_num #print("THE dbsv_freq is {}".format(dbsv_freq)) #print(filter_sample_list) #print(len(filter_sample_list)) ##频率:基于dbSV中正常人样本 healthy_sample_list = [] healthy_num = 0 for sample in filter_sample_list: #if sample in healthy_db_list: if sample.split(":")[0] in healthy_db_list: healthy_num += 1 healthy_sample_list.append( sample) #get healthy samples from filter_sample_list dbsv_healthy_freq = healthy_num / healthy_db_num #print("The dbsv_healthy_freq is {}".format(dbsv_healthy_freq)) ender = time.time() timer = ender - starter #print(healthy_sample_list) #print(len(healthy_sample_list)) #out_str = "\t".join([str(dbsv_freq),str(dbsv_healthy_freq),str(timer)])+"\n" out_str = "" if len(filter_sample_list) != 0: out_str = "\t".join([ target_svid, str(dbsv_healthy_freq), str(dbsv_freq), ";".join(filter_sample_list) ]) + "\n" else: out_str = "\t".join( [target_svid, str(dbsv_healthy_freq), str(dbsv_freq), "."]) + "\n" out_list.append(out_str) out_io.write("\t".join([ "#SVID", "GrandSV_Healthy_Frequency", "GrandSV_Frequency", "GrandSV_LIST" ]) + "\n") out_io.writelines(out_list) b_time = time.time() total_time = b_time - a_time
from datetime import datetime from elasticsearch6 import Elasticsearch es = Elasticsearch({"172.16.100.186": "9200"}) doc = {'id': 2, 'name': "乙"} ID = 2 INDEX = "wuren" DOC_TYPE = "cn" # res = es.delete(index=INDEX, doc_type=DOC_TYPE, id=ID) es.create(index=INDEX, doc_type=DOC_TYPE, id=ID, body=doc) # res = es.delete(index=INDEX, doc_type=DOC_TYPE, id=2) res = es.get(index=INDEX, doc_type=DOC_TYPE, id=ID) print(res) # res = es.get(index=InterruptedError, id=1, doc_type=DOC_TYPE) # print(res['_source'])
def get_cluster_stats(): # Count master nodes number_of_master_nodes = 0 number_of_ingest_nodes = 0 number_of_master_data_nodes = 0 for node_name in node_names: #print("\t\t" + node_name['id'] + "\t" + node_name['ip'] + "\n") if "m" in node_name['node.role']: number_of_master_nodes += 1 master_node = True if "d" in node_name['node.role']: data_node = True if "i" in node_name['node.role']: number_of_ingest_nodes += 1 if master_node and data_node: number_of_master_data_nodes += 1 # Fetch all stats es = Elasticsearch(es_url, verify_certs=False, connection_class=RequestsHttpConnection) cluster_health = es.cluster.health() indices_cat = es.cat.indices(format="json", bytes="b") total_indices = 0 total_green_indices = 0 total_open_indices = 0 total_docs = 0 total_size = 0 for index in indices_cat: total_indices += 1 total_docs += int(index["docs.count"]) total_size += int(index["store.size"]) if index["health"] == "green": total_green_indices += 1 if index["status"] == "open": total_open_indices += 1 total_non_green_indices = total_indices - total_green_indices total_closed_indices = total_indices - total_green_indices cluster_name = cluster_health['cluster_name'] cluster_status = cluster_health['status'] if cluster_status == 'green': cluster_status_num = 100 elif cluster_status == 'red': cluster_status_num = 0 elif cluster_status == 'yellow': cluster_status_num = 50 else: cluster_status_num = 100 number_of_data_nodes = cluster_health['number_of_data_nodes'] number_of_nodes = cluster_health['number_of_nodes'] active_primary_shards = cluster_health['active_primary_shards'] active_shards = cluster_health['active_shards'] active_shards_percent_as_number = cluster_health[ 'active_shards_percent_as_number'] data = "ClusterStats,ClusterName=" + cluster_name + \ " ClusterStatus=" + str(cluster_status_num) + \ ",number_of_nodes=" + str(number_of_nodes) + \ ",number_of_data_nodes=" + str(number_of_data_nodes) + \ ",number_of_master_nodes=" + str(number_of_master_nodes) + \ ",active_primary_shards=" + str(active_primary_shards) + \ ",active_shards=" + str(active_shards) + \ ",active_shards_percent_as_number=" + str(active_shards_percent_as_number) + \ ",total_index=" + str(total_indices) + \ ",total_healthy_index=" + str(total_green_indices) + \ ",total_non_green_indices=" + str(total_non_green_indices) + \ ",total_open_indices=" + str(total_open_indices) + \ ",total_closed_indices=" + str(total_closed_indices) + \ ",total_docs=" + str(total_docs) + \ ",total_size=" + str(total_size) print("[INFO] [" + datetime.now().strftime('%Y-%m-%d %H:%M:%S') + "]: Uploading Custer Stats to Influx") print("\t\t[DATA]" + data) resp = requests.post(influxdb_url, data=data) print("\t\tResponse from Influx = " + str(resp.status_code) + "\n")
# get created date for a repo and fallback to authored_date for a commit created_at = parse_date(hit['_source'].get('created_at', hit['_source']['authored_date'])) print('/%s/%s/%s (%s): %s' % ( hit['_index'], hit['_type'], hit['_id'], created_at.strftime('%Y-%m-%d'), hit['_source']['description'].split('\n')[0])) print('=' * 80) print() # get trace logger and set level tracer = logging.getLogger('elasticsearch.trace') tracer.setLevel(logging.INFO) tracer.addHandler(logging.FileHandler('/tmp/es_trace.log')) # instantiate es client, connects to localhost:9200 by default es = Elasticsearch() print('Empty search:') print_hits(es.search(index='git')) print('Find commits that says "fix" without touching tests:') result = es.search( index='git', doc_type='doc', body={ 'query': { 'bool': { 'must': { 'match': {'description': 'fix'} }, 'must_not': {
import json import re from datetime import datetime from elasticsearch6 import Elasticsearch if __name__ == '__main__': elasticsearch = Elasticsearch(hosts=['192.168.1.68:9200']) elasticsearch.indices.create(index='imdb', ignore=400) with open('movies.jl', 'r') as file: lines = file.readlines() for line in lines: data = json.loads(line) casts = [ f'{key.replace("-", "").strip()}-{val.replace("-", "").strip()}' for key, val in data['cast'].items() ] try: release_date, place = re.findall(r'(.+)\s\((.+)\)', data['release_date'])[0] data['release_date'] = str( datetime.strptime(release_date, '%d %B %Y').replace(day=1).date()) except ValueError: try: data['release_date'] = str( datetime.strptime(release_date, '%B %Y').date()) except ValueError: data['release_date'] = str( datetime.strptime(release_date, '%Y').date()) except KeyError: place = 'Unknown'
def setUp(self): super(ElasticsearchTestCase, self).setUp() self.client = Elasticsearch(transport_class=DummyTransport)
help= "The elasticsearch host you wish to connect too. (Default: localhost:9200)" ) parser.add_argument( "-p", "--path", action="store", default=None, help= "Path to git repo. Commits used as data to load into Elasticsearch. (Default: None" ) args = parser.parse_args() # instantiate es client, connects to localhost:9200 by default es = Elasticsearch(args.host) # we load the repo and all commits load_repo(es, path=args.path) # run the bulk operations success, _ = bulk(es, UPDATES, index='git') print('Performed %d actions' % success) # we can now make docs visible for searching es.indices.refresh(index='git') # now we can retrieve the documents initial_commit = es.get(index='git', doc_type='doc', id='20fbba1230cabbc0f4644f917c6c2be52b8a63e8')