def get_json(): list_json = [] for i in os.listdir(CONST.JSON_PATH): if i.endswith('.json'): log.info('read json_files files:{}'.format(i)) with open(os.path.join(CONST.JSON_PATH, i), 'r') as f: str_value = f.read() list_json.append({'value': loads(str_value), 'file_name': i[:-5]}) return list_json
def create_table(self, table_name, families): try: with self.connection_pool.connection(self.timeout) as connection: connection.create_table(table_name, families) log.info( 'target table: {} create successfully.'.format(table_name)) except Exception as e: log.error('target table: {} create fail.'.format( table_name, str(e))) raise Exception(str(e))
def __init__(self, host=DEFAULT_HOST, port=DEFAULT_PORT, timeout=None, autoconnect=True): self.hbase_host = host self.hbase_port = port self.timeout = timeout self.autoconnect = autoconnect self.connection_pool = happybase.ConnectionPool( CONST.HAPPYBASE_POOL_SIZE, host=self.hbase_host, port=self.hbase_port, autoconnect=True) self.reconnect_count = 0 log.info('init hbase: {}'.format(host))
def de_initialize(self): try: log.info('try to de-initialize the system management.') if not self.has_been_initializer: log.info('base has not been initialized.') return self.has_been_initializer = False log.info('base de initialize successfully.') except Exception as err: log.info('stop base initializer error {}'.format(err))
def initialize(self): log.info('try to initialize the base wrappers.') if self.has_been_initializer: log.info('system management has been initialized yet.') return # Attention: Don't change the order for initializing component. # env --> etcd --> setting --> redis --> stat self._init_redis() self._init_mysql() self._init_hdfs() self._init_hbase() log.info('base initialize successfully.') self.has_been_initializer = True
def create_tables(self, base): try: log.info('creating sql tables.') base.metadata.create_all(self.engine) except Exception as e: log.error('create tables failed {}'.format(str(e)))
def __init__(self): self.session = None self.engine = None log.info('init mysql: {}'.format(CONST.DB_URL))