def get_es_conn(): """Return an Elasticsearch ConnectionContext.""" cm = ConfigurationManager( ConnectionContext.get_required_config(), values_source_list=[environment] ) config = cm.get_config() return ConnectionContext(config)
def get_conn(): # Create a configuration manager that will only check the environment for # configuration and not command line parameters cm = ConfigurationManager(ConnectionContext.get_required_config(), values_source_list=[environment]) config = cm.get_config() return ConnectionContext(config)
def handle(self, **options): config = config_from_configman()["elasticsearch"] conn = ConnectionContext(config) indices = conn.delete_expired_indices() if indices: self.stdout.write("Deleting expired crash report indices.") for index in indices: self.stdout.write("Deleting %s" % index) else: self.stdout.write("No expired indices to delete.")
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards.""" cm = ConfigurationManager( ConnectionContext.get_required_config(), values_source_list=[environment] ) config = cm.get_config() conn = ConnectionContext(config) yield conn for index in conn.get_indices(): conn.delete_index(index)
def setup_method(self, method): super().setup_method(method) self.config = self.get_base_config() self.es_context = ConnectionContext(self.config) self.index_client = self.es_context.indices_client() with self.es_context() as conn: self.connection = conn self.es_context.create_index(self.es_context.get_index_template())
def test_connection_context_client(self): """Instantiate the client and ensure that it quacks like a duck. """ es_context = ConnectionContext(config=self.config.elasticsearch) client = es_context.connection() # The client *must* have specific elements. ok_(client._connection) ok_(client.close) ok_(client.commit) ok_(client.config) ok_(client.rollback) # The underlying ES interface is exposed by _connection. This API is # exhaustive and well outside of the scope of this test suite; in the # interest of safety however, we'll check one here. ok_(client._connection.index)
def get_conn(): # Create a configuration manager that will only check the environment for # configuration and not command line parameters cm = ConfigurationManager( ConnectionContext.get_required_config(), values_source_list=[environment] ) config = cm.get_config() return ConnectionContext(config)
def setup_method(self, method): super().setup_method(method) self.config = self.get_base_config() self.es_context = ConnectionContext(self.config) self.index_client = self.es_context.indices_client() with self.es_context() as conn: self.connection = conn self.es_context.create_socorro_index(self.es_context.get_index_template())
def setup_method(self): super().setup_method() self.config = self.get_base_config() self.es_context = ConnectionContext(self.config) self.crashstorage = ESCrashStorage( config=self.get_tuned_config(ESCrashStorage)) self.index_client = self.es_context.indices_client() self.conn = self.es_context.connection() # Delete everything there first for index_name in self.es_context.get_indices(): print(f"setup: delete test index: {index_name}") self.es_context.delete_index(index_name) to_create = [ self.es_context.get_index_for_date(utc_now()), self.es_context.get_index_for_date(utc_now() - timedelta(days=7)), ] for index_name in to_create: print(f"setup: creating index: {index_name}") self.es_context.create_index(index_name)
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards.""" cm = ConfigurationManager(ConnectionContext.get_required_config(), values_source_list=[environment]) config = cm.get_config() conn = ConnectionContext(config) yield conn for index in conn.get_indices(): conn.delete_index(index)
def test_connection_context(self): """Instantiate the context and ensure that it quacks like a duck. """ # The context is effectively a connection factory. es_context = ConnectionContext(config=self.config.elasticsearch) # The connection context *must* have specific elements. assert es_context.config assert es_context.connection # There is one operational exception. assert elasticsearch.exceptions.ConnectionError in es_context.operational_exceptions # Currently there are no conditional exceptions. assert len(es_context.conditional_exceptions) == 0
class ElasticsearchTestCase(TestCaseWithConfig): """Base class for Elastic Search related unit tests""" def setup_method(self, method): super().setup_method(method) self.config = self.get_base_config() self.es_context = ConnectionContext(self.config) self.index_client = self.es_context.indices_client() with self.es_context() as conn: self.connection = conn self.es_context.create_socorro_index(self.es_context.get_index_template()) def teardown_method(self, method): # Clear the test indices. self.index_client.delete(self.es_context.get_index_template()) super().teardown_method(method) def health_check(self): self.connection.cluster.health(wait_for_status='yellow', request_timeout=5) def get_url(self): """Returns the first url in the elasticsearch_urls list""" return self.config.elasticsearch_urls[0] def get_tuned_config(self, sources, extra_values=None): values_source = DEFAULT_VALUES.copy() if extra_values: values_source.update(extra_values) return super().get_tuned_config(sources, values_source) def get_base_config(self, cls=ConnectionContext, es_index=None): extra_values = None if es_index: extra_values = { 'resource.elasticsearch.elasticsearch_index': es_index } return self.get_tuned_config(cls, extra_values=extra_values) def index_crash(self, processed_crash=None, raw_crash=None, crash_id=None): if crash_id is None: crash_id = str(uuid.UUID(int=random.getrandbits(128))) raw_crash = raw_crash or {} processed_crash = processed_crash or {} doc = { 'crash_id': crash_id, 'processed_crash': processed_crash, 'raw_crash': raw_crash, } index_name = self.es_context.get_index_template() res = self.connection.index( index=index_name, doc_type=self.es_context.get_doctype(), id=crash_id, body=doc, ) return res['_id'] def index_many_crashes(self, number, processed_crash=None, raw_crash=None, loop_field=None): processed_crash = processed_crash or {} raw_crash = raw_crash or {} actions = [] for i in range(number): crash_id = str(uuid.UUID(int=random.getrandbits(128))) if loop_field is not None: processed_copy = processed_crash.copy() processed_copy[loop_field] = processed_crash[loop_field] % i else: processed_copy = processed_crash doc = { 'crash_id': crash_id, 'processed_crash': processed_copy, 'raw_crash': raw_crash, } action = { '_index': self.es_context.get_index_template(), '_type': self.es_context.get_doctype(), '_id': crash_id, '_source': doc, } actions.append(action) bulk(client=self.connection, actions=actions) self.refresh_index() def refresh_index(self, es_index=None): self.index_client.refresh(index=es_index or self.es_context.get_index_template())
class ElasticsearchTestCase(TestCaseWithConfig): """Base class for Elastic Search related unit tests""" def setup_method(self, method): super().setup_method(method) self.config = self.get_base_config() self.es_context = ConnectionContext(self.config) self.index_client = self.es_context.indices_client() with self.es_context() as conn: self.connection = conn self.es_context.create_index(self.es_context.get_index_template()) def teardown_method(self, method): # Clear the test indices. self.index_client.delete(self.es_context.get_index_template()) super().teardown_method(method) def health_check(self): self.connection.cluster.health(wait_for_status="yellow", request_timeout=5) def get_url(self): """Returns the first url in the elasticsearch_urls list""" return self.config.elasticsearch_urls[0] def get_tuned_config(self, sources, extra_values=None): values_source = DEFAULT_VALUES.copy() if extra_values: values_source.update(extra_values) return super().get_tuned_config(sources, values_source) def get_base_config(self, cls=ConnectionContext, es_index=None): extra_values = None if es_index: extra_values = { "resource.elasticsearch.elasticsearch_index": es_index } return self.get_tuned_config(cls, extra_values=extra_values) def index_crash(self, processed_crash=None, raw_crash=None, crash_id=None): if crash_id is None: crash_id = str(uuid.UUID(int=random.getrandbits(128))) raw_crash = raw_crash or {} processed_crash = processed_crash or {} doc = { "crash_id": crash_id, "processed_crash": processed_crash, "raw_crash": raw_crash, } index_name = self.es_context.get_index_template() res = self.connection.index( index=index_name, doc_type=self.es_context.get_doctype(), id=crash_id, body=doc, ) return res["_id"] def index_many_crashes(self, number, processed_crash=None, raw_crash=None, loop_field=None): processed_crash = processed_crash or {} raw_crash = raw_crash or {} actions = [] for i in range(number): crash_id = str(uuid.UUID(int=random.getrandbits(128))) if loop_field is not None: processed_copy = processed_crash.copy() processed_copy[loop_field] = processed_crash[loop_field] % i else: processed_copy = processed_crash doc = { "crash_id": crash_id, "processed_crash": processed_copy, "raw_crash": raw_crash, } action = { "_index": self.es_context.get_index_template(), "_type": self.es_context.get_doctype(), "_id": crash_id, "_source": doc, } actions.append(action) bulk(client=self.connection, actions=actions) self.es_context.refresh()
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards.""" conn = ConnectionContext(config_from_configman()['elasticsearch']) yield conn for index in conn.get_indices(): conn.delete_index(index)
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards. This uses defaults and configuration from the environment. """ manager = ConfigurationManager(ESConnectionContext.get_required_config(), values_source_list=[environment]) conn = ESConnectionContext(manager.get_config()) # Create two indexes--this week and last week template = conn.config.elasticsearch_index conn.create_index(utc_now().strftime(template)) conn.create_index( (utc_now() - datetime.timedelta(weeks=1)).strftime(template)) conn.health_check() yield conn for index in conn.get_indices(): conn.delete_index(index)
def es_conn(): """Create an Elasticsearch ConnectionContext and clean up indices afterwards.""" conn = ConnectionContext(config_from_configman()["elasticsearch"]) yield conn for index in conn.get_indices(): conn.delete_index(index)
class ElasticsearchTestCase(TestCaseWithConfig): """Base class for Elastic Search related unit tests""" def setup_method(self): super().setup_method() self.config = self.get_base_config() self.es_context = ConnectionContext(self.config) self.crashstorage = ESCrashStorage( config=self.get_tuned_config(ESCrashStorage)) self.index_client = self.es_context.indices_client() self.conn = self.es_context.connection() # Delete everything there first for index_name in self.es_context.get_indices(): print(f"setup: delete test index: {index_name}") self.es_context.delete_index(index_name) to_create = [ self.es_context.get_index_for_date(utc_now()), self.es_context.get_index_for_date(utc_now() - timedelta(days=7)), ] for index_name in to_create: print(f"setup: creating index: {index_name}") self.es_context.create_index(index_name) def teardown_method(self): for index_name in self.es_context.get_indices(): print(f"teardown: delete test index: {index_name}") self.es_context.delete_index(index_name) super().teardown_method() def health_check(self): self.conn.cluster.health(wait_for_status="yellow", request_timeout=5) def get_url(self): """Returns the first url in the elasticsearch_urls list""" return self.config.elasticsearch_urls[0] def get_tuned_config(self, sources, extra_values=None): values_source = DEFAULT_VALUES.copy() if extra_values: values_source.update(extra_values) return super().get_tuned_config(sources, values_source) def get_base_config(self, cls=ConnectionContext, es_index=None): extra_values = None if es_index: extra_values = { "resource.elasticsearch.elasticsearch_index": es_index } return self.get_tuned_config(cls, extra_values=extra_values) def index_crash(self, processed_crash=None, raw_crash=None, crash_id=None, refresh=True): """Index a single crash and refresh""" if crash_id is None: crash_id = str(uuid.UUID(int=random.getrandbits(128))) raw_crash = raw_crash or {} processed_crash = processed_crash or {} raw_crash["uuid"] = crash_id processed_crash["crash_id"] = crash_id processed_crash["uuid"] = crash_id self.crashstorage.save_processed_crash(raw_crash, processed_crash) if refresh: self.es_context.refresh() return crash_id def index_many_crashes(self, number, processed_crash=None, raw_crash=None, loop_field=None): """Index multiple crashes and refresh at the end""" processed_crash = processed_crash or {} raw_crash = raw_crash or {} crash_ids = [] for i in range(number): if loop_field is not None: processed_copy = processed_crash.copy() processed_copy[loop_field] = processed_crash[loop_field] % i else: processed_copy = processed_crash crash_ids.append( self.index_crash(raw_crash=raw_crash, processed_crash=processed_copy, refresh=False)) self.es_context.refresh() return crash_ids