def handle(self, *args, **options): assert settings.DEBUG, "Only ever works in DEBUG mode" if input("Are you sure? [y/N] ").lower().strip() != 'y': print("Aborted!") return Build.objects.all().delete() build_index.delete(ignore=404)
def handle(self, *args, **options): build_index.delete(ignore=404) build_index.create() es_url = settings.ES_URLS[0] index_name = settings.ES_BUILD_INDEX update_settings_url = f"{es_url}/{index_name}/_settings" response = requests.put(update_settings_url, json={"index": { "refresh_interval": "-1" }}) response.raise_for_status() es = connections.get_connection() report_every = 1000 count = 0 qs = Build.objects.all().order_by("created_at") total_count = qs.count() iterator = qs.iterator(chunk_size=10000) for success, doc in streaming_bulk( es, (m.to_search().to_dict(True) for m in iterator), index=index_name, doc_type="doc", ): if not success: raise Exception(doc) count += 1 if not count % report_every: print( format(count, ",").ljust(8), "\t", "{:.1f}%".format(100 * count / total_count), ) # We have done a bunch of bulk inserts into Elasticsearch with # the refresh interval "disabled". Now we need to "force merge" # the index. See # https://www.elastic.co/guide/en/elasticsearch/reference/6.x/indices-update-settings.html#bulk force_merge_url = f"{es_url}/{index_name}/_forcemerge?max_num_segments=5" response = requests.post(force_merge_url) response.raise_for_status() print("Force merged Elasticsearch index after bulk insert") # Restore refresh_interval response = requests.put( update_settings_url, json={ "index": { "refresh_interval": settings.ES_BUILD_INDEX_SETTINGS["refresh_interval"] } }, ) response.raise_for_status()
def elasticsearch(request): """Returns the index object for builds. But before, it re-creates the index. And after the index is deleted. All 404s ignored. Usage:: def test_something(elasticsearch): # Test stuff and assume the Elasticsearch build index # exists and is empty. assert something """ assert build_index._name.startswith("test_") build_index.delete(ignore=404) build_index.create() yield build_index build_index.delete(ignore=404)