コード例 #1
0
    def run(self):
        print("\n*********** STARTING... ***********\n")
        print('ES INDEX: {0}'.format(self.es_index))
        page = self.get_page()
        sid = page['_scroll_id']
        scroll_size = page['hits']['total']

        progress = ProgressBar(scroll_size, 'completed')
        progress.print()

        self.silent_remove(self.file_path)
        self.write_header_to_csv(self.file_path, parser.get_column_names(self.body))
        self.write_data_to_csv(page['hits']['hits'])
        while (scroll_size > 0):
            page = self.es.scroll(scroll_id=sid, scroll='2m')
            sid = page['_scroll_id']
            scroll_size = len(page['hits']['hits'])
            progress.update_current_count(scroll_size)
            progress.print()
            self.write_data_to_csv(page['hits']['hits'])
        print("\n*********** COMPLETED ***********")