def execute_query(self): if self.DB_QUERY is None: ES_LOGGER.warning("数据库语句空缺") try: self.DB_CURSOR.execute(self.DB_QUERY, self.DB_ARGS) except Exception, e: self.DB_CONNECT.rollback() ES_LOGGER.debug("数据库执行错误:%s " % e)
def bulk_elasticsearch(r_queue, w_lock, dbs, db_name): ES_LOGGER.info("Bulk Host: %s DB: %s Start" % (dbs['db_host'], db_name)) es = Elasticsearch(dbs['es_colony'], retry_on_timeout=True, max_retries=3, timeout=3600) flag = True bulks = [] data_lines_number = 0 bulk_length = 0 while flag: while not r_queue.empty(): if bulk_length == 0: w_lock.acquire() data = r_queue.get() data_lines_number += 1 bulk_length += 1 if bulk_length >= BULK_LENGTH or r_queue.empty(): w_lock.release() if isinstance(data, str) and data == 'False': try: ES_LOGGER.info("Bulk Host: %s DB: %s Data: %s" % (dbs['db_host'], db_name, bulk_length)) streaming_bulks = helpers.streaming_bulk(es, bulks, chunk_size=len(bulks)) for streaming_bulk in streaming_bulks: if streaming_bulk[0]: pass bulks = [] except Exception, e: ES_LOGGER.warning(e) flag = False break bulks.append({ "_index": dbs['index'], "_type": dbs['doc_type'], "_source": data }) if bulk_length >= BULK_LENGTH: try: ES_LOGGER.info("Bulk Host: %s DB: %s Data: %s" % (dbs['db_host'], db_name, data_lines_number),) streaming_bulks = helpers.streaming_bulk(es, bulks, chunk_size=len(bulks)) for streaming_bulk in streaming_bulks: if streaming_bulk[0]: pass bulks = [] bulk_length = 0 except Exception, e: ES_LOGGER.warning("Bulk Error! %s", e)
def bulk_elasticsearch(r_queue, w_lock, dbs, db_name): ES_LOGGER.info("Bulk Host: %s DB: %s Start" % (dbs["db_host"], db_name)) es = Elasticsearch(dbs["es_colony"], retry_on_timeout=True, max_retries=3, timeout=3600) flag = True bulks = [] data_lines_number = 0 bulk_length = 0 while flag: while not r_queue.empty(): if bulk_length == 0: w_lock.acquire() data = r_queue.get() data_lines_number += 1 bulk_length += 1 if bulk_length >= BULK_LENGTH or r_queue.empty(): w_lock.release() if isinstance(data, str) and data == "False": try: ES_LOGGER.info("Bulk Host: %s DB: %s Data: %s" % (dbs["db_host"], db_name, bulk_length)) streaming_bulks = helpers.streaming_bulk(es, bulks, chunk_size=len(bulks)) for streaming_bulk in streaming_bulks: if streaming_bulk[0]: pass bulks = [] except Exception, e: ES_LOGGER.warning(e) flag = False break bulks.append({"_index": dbs["index"], "_type": dbs["doc_type"], "_source": data}) if bulk_length >= BULK_LENGTH: try: ES_LOGGER.info("Bulk Host: %s DB: %s Data: %s" % (dbs["db_host"], db_name, data_lines_number)) streaming_bulks = helpers.streaming_bulk(es, bulks, chunk_size=len(bulks)) for streaming_bulk in streaming_bulks: if streaming_bulk[0]: pass bulks = [] bulk_length = 0 except Exception, e: ES_LOGGER.warning("Bulk Error! %s", e)