def handle(self, *args, **options): min_id = FailureLine.objects.order_by('id').values_list("id", flat=True)[0] - 1 chunk_size = options['chunk_size'] if options["recreate"]: connection.indices.delete(TestFailureLine._doc_type.index, ignore=404) TestFailureLine.init() else: if connection.indices.exists(TestFailureLine._doc_type.index): self.stderr.write("Index already exists; can't perform import") return while True: rows = (FailureLine.objects .filter(id__gt=min_id) .order_by('id') .values("id", "job_guid", "action", "test", "subtest", "status", "expected", "message", "best_classification_id", "best_is_verified"))[:chunk_size] if not rows: break es_lines = [] for item in rows: es_line = failure_line_from_value(item) if es_line: es_lines.append(es_line) self.stdout.write("Inserting %i rows" % len(es_lines)) bulk_insert(es_lines) min_id = rows[len(rows) - 1]["id"] time.sleep(options['sleep']) s = Search(doc_type=TestFailureLine).params(search_type="count") self.stdout.write("Index contains %i documents" % s.execute().hits.total)
def create_es(failure_lines): # Store the failure lines in elastic_search es_lines = [] for failure_line in failure_lines: es_line = TestFailureLine.from_model(failure_line) if es_line: es_lines.append(es_line) bulk_insert(es_lines)
def create_es(failure_lines): # Store the failure lines in elastic_search es_lines = [] for failure_line in failure_lines: es_line = TestFailureLine.from_model(failure_line) if es_line: es_lines.append(es_line) bulk_insert(es_lines)
def handle(self, *args, **options): if options["recreate"]: connection.indices.delete(TestFailureLine._doc_type.index, ignore=404) TestFailureLine.init() elif connection.indices.exists(TestFailureLine._doc_type.index): self.stderr.write("Index already exists; can't perform import") return fields = [ 'id', 'action', 'job_guid', 'test', 'subtest', 'status', 'expected', 'message', 'best_classification_id', 'best_is_verified', ] failure_lines = FailureLine.objects.filter(action='test_result') for rows in chunked_qs(failure_lines, options['chunk_size'], fields=fields): if not rows: break es_lines = [TestFailureLine.from_model(line) for line in rows] self.stdout.write("Inserting %i rows" % len(es_lines)) bulk_insert(es_lines) time.sleep(options['sleep']) count = Search(doc_type=TestFailureLine).count() self.stdout.write("Index contains %i documents" % count)