def connection(self): ctx = stack.top if ctx is not None: if not hasattr(ctx, 'esclient_connection'): ctx.esclient_connection = esclient.ESClient( current_app.config['ELASTICSEARCH_URL']) return ctx.esclient_connection
def main(): config = parse_args(sys.argv[1:]) print config es_client = es.ESClient(hosts=config.hosts.split(","), timeout=30) print "\nVerifying counts..." tenant_ids = [] if config.tenantIds: tenant_ids = config.tenantIds.split(',') tenant_id_infos = count_index(es_client, config.current_index, "metrics", tenant_ids) if not config.dryrun: print "\nReindexing..." for tenantId in tenant_id_infos: print "\n***** Re-indexing tenantId: {0} started at: {1}".format( tenantId[0], datetime.datetime.now()) es_client.reindex(tenantId[0], config.current_index, config.new_index, config.new_index_type, config.scroll_timeout, config.size, config.transform, config.bulk_thread_count, config.bulk_size) print "***** Re-indexing tenantId: {0} ended at : {1}".format( tenantId[0], datetime.datetime.now())
def setUpClass(self): """Create an ESClient""" self.es = esclient.ESClient() """Delete the test schema, if any. This will prevent any errors due to the schema already existing """ print("Deleting test indexes, if any") self.es.delete_index("contacts_esclient_test") self.es.delete_index("contacts_esclient_test2")
def main(): es_object = esclient.ESClient("http://*****:*****@gmail.com AND amazon'} result = es_object.search(query_string_args=query_string_args, indexes=['sent_counts']) #print_dict(result) print result['hits']['total']
def elasticsearch(es_port,es_host): #Create an ESClient global clientes clientes = esclient.ESClient("http://" + es_host + ":" + es_port + "/") #Parse all indices for inputs in configSectionValue('input'): # create index logger.debug("creating index: %s", inputs) clientes.create_index(inputs,body=None)
def main(): args = parse_arguments(sys.argv[1:]) config = cf.Config(args.env.lower()) db_client = db.DBClient() db_client.connect(config.get_cassandra_nodes()) es_client = es.ESClient(config.get_es_nodes()) # 'delete' command has the namespace: Namespace(dryrun=True, metricName='metric_name', tenantId='tenant_id') if 'dryrun' in args: clear_excess_enums(args, db_client, es_client) else: list_excess_enums(db_client)
def main(): LOGGER.info("Starting application") parser = argparse.ArgumentParser() parser.add_argument("--elasticsearch", help="Base URL to contact ElasticSearch", type=str, required=True) parser.add_argument("--indexprefix", help="ElasticSearch Index Prefix", default=INDEX_PREFIX, type=str) parser.add_argument("--hours-ahead", help="How many hours in advance", default=MAX_AGE_IN_HOURS, type=int) args = parser.parse_args() elasticsearch_url = args.elasticsearch index_prefix = args.indexprefix hours_ahead = args.hours_ahead LOGGER.info("Contacting ElasticSearch for status: [%s]" % (elasticsearch_url)) es_connection = esclient.ESClient(elasticsearch_url) now = datetime.utcnow() for hour in xrange(hours_ahead): hour_delta = timedelta(hours=hour) hour_date = now + hour_delta index_name = index_prefix + INDEX_SEPARATOR + "%.4d.%.2d.%.2d-%.2d" % ( hour_date.year, hour_date.month, hour_date.day, hour_date.hour) retry = 0 success = False while retry < MAX_CREATE_RETRY and not success: try: LOGGER.info("Creating index [%s]" % (index_name)) es_connection.create_index(index_name) LOGGER.debug("Done creating index [%s]" % (index_name)) success = True except Exception, e: LOGGER.error('Could not create index %s: [%s]' % (index_name, str(e))) finally:
def main(): LOGGER.info("Starting application") parser = argparse.ArgumentParser() parser.add_argument("--elasticsearch", help="Base URL to contact ElasticSearch", type=str, required=True) parser.add_argument("--ttl", help="Maximum age in days", default=MAX_AGE_IN_DAYS, type=int) parser.add_argument("--index", help="Name of the index", default=DEFAULT_INDEX_NAME, type=str, required=False) args = parser.parse_args() max_ttl_seconds = args.ttl * 24 * 3600 elasticsearch_url = args.elasticsearch index_name = args.index LOGGER.info("Contacting ElasticSearch for status: [%s]" % (elasticsearch_url)) es_connection = esclient.ESClient(elasticsearch_url) status = es_connection.status() LOGGER.info("ElasticSearch status retrieved") now = datetime.utcnow() if INDICES_KEY in status: for index in status[INDICES_KEY]: try: LOGGER.debug("Processing index [%s]" % (index)) split_index = index.split(INDEX_SEPARATOR) if split_index and len(split_index) > 2: split = index.split(INDEX_SEPARATOR) logstash = INDEX_SEPARATOR.join(split[:-2]) date_str = split[-2] shard = split[-1] if logstash == index_name: date = datetime.strptime(date_str, '%Y.%m.%d') age_date = now - date age_seconds = age_date.days * 24 * 3600 + age_date.seconds if age_seconds > max_ttl_seconds: retry = 0 success = False while retry < MAX_DELETE_RETRY and not success: try: LOGGER.info( "Deleting index [%s] since it is older than %d days" % (index, args.ttl)) success = es_connection.delete_index(index) LOGGER.debug("Done deleting index [%s]" % (index)) except Exception, e: LOGGER.error( 'Could not delete index %s: [%s]' % (index, str(e))) finally: retry = retry + 1
def setUpClass(cls): """Create an ESClient""" cls.es = esclient.ESClient("http://localhost:9200/") """Delete the test schema, if any. This will prevent any errors due to the schema already existing """ cls.es.delete_index("contacts_esclient_test")