class ESServer(object): def __init__(self, url): """ Initialize ES Server :param url: ES URL """ self.server = Elasticsearch([url]) self.batch = 100 self.index = '' def upload(self, docs): actions = [] for doc in docs: doc_id = doc['id'] del doc['id'] action = { "_index": self.index, "_type": "doc", "_id": doc_id, "_source": doc } actions.append(action) helpers.bulk(self.server, actions) def commit(self): self.server.commit() pass
class DBConnection(object): """ Class DBConnection: connect db """ MYSQL = 1 MSSQL = 2 IMPALA = 3 ORACLE = 4 HIVE = 5 ES = 6 def __init__(self, **kwargs): """ """ self.host = kwargs.get('host', 'localhost') self.database = kwargs.get('database') self.engine = kwargs.get('engine', self.MYSQL) self.conn = None if self.engine is None or self.engine == self.MYSQL: self.user = kwargs.get('user') self.password = kwargs.get('password') self.charset = kwargs.get('charset', 'utf8') self.port = kwargs.get('port', 3306) if self.user and self.password: from pymysql import connect self.conn = connect(host=self.host, port=int(self.port), user=self.user, password=self.password, database=self.database, charset=self.charset) elif self.engine == self.MSSQL: self.user = kwargs.get('user') self.password = kwargs.get('password') self.charset = kwargs.get('charset', 'utf8') self.port = kwargs.get('port', 2521) if self.user and self.password: from pymssql import connect if not self.port: self.port = 2521 self.conn = connect(host=self.host, user=self.user, password=self.password, database=self.database, charset=self.charset) elif self.engine == self.IMPALA: self.backup_host = kwargs.get('backup') self.port = kwargs.get('port', 21050) if self.host: if not self.port: self.port = 21050 from impala.dbapi import connect try: self.conn = connect(self.host, int(self.port), self.database) except Exception as ex: if self.backup_host: print( 'connect to server:%s failed, try the backup: %s' % (self.host, self.backup_host)) try: self.conn = connect(self.backup_host, int(self.port), self.database) except Exception as ex: raise ex elif self.engine == self.ORACLE: # self.user = kwargs.get('user') # self.password = kwargs.get('password') # self.charset = kwargs.get('charset', 'utf8') # self.instance = kwargs.get('instance') # self.port = kwargs.get('port', 1125) # from cx_Oracle import connect # self.connect_cmd = "%s/%s@%s:%s/%s" %( # self.user, # self.password, # self.host, # int(self.port), # self.instance) # self.conn = connect(self.connect_cmd) pass elif self.engine == self.HIVE: # self.port = kwargs.get('port', 10000) # self.authMechanism = kwargs.get('authMechanism', 'PLAIN') # self.user = kwargs.get('user') # self.password = kwargs.get('password') # self.conn = connect(host=self.host, port=self.port, # authMechanism=self.authMechanism, user=self.user, # password=self.password, database=self.database) pass elif self.engine == self.ES: self.port = kwargs.get('port', 10000) from elasticsearch import Elasticsearch self.conn = Elasticsearch([{'host': self.host, 'port': self.port}]) else: self.conn = None def query(self, sql): if sql and self.conn: try: data = None with self.conn.cursor() as cursor: cursor.execute(sql) data = cursor.fetchall() import re if re.findall(r'[insert|update|delete]', sql.lower()): self.conn.commit() return True, data except Exception as ex: raise ex def __del__(self): if self.conn: try: self.conn.close() except Exception as ex: print(ex)