def get_json():
list_json = []
for i in os.listdir(CONST.JSON_PATH):
if i.endswith('.json'):
log.info('read json_files files:{}'.format(i))
with open(os.path.join(CONST.JSON_PATH, i), 'r') as f:
str_value = f.read()
list_json.append({'value': loads(str_value), 'file_name': i[:-5]})
return list_json
def create_table(self, table_name, families):
try:
with self.connection_pool.connection(self.timeout) as connection:
connection.create_table(table_name, families)
log.info(
'target table: {} create successfully.'.format(table_name))
except Exception as e:
log.error('target table: {} create fail.'.format(
table_name, str(e)))
raise Exception(str(e))
def __init__(self,
host=DEFAULT_HOST,
port=DEFAULT_PORT,
timeout=None,
autoconnect=True):
self.hbase_host = host
self.hbase_port = port
self.timeout = timeout
self.autoconnect = autoconnect
self.connection_pool = happybase.ConnectionPool(
CONST.HAPPYBASE_POOL_SIZE,
host=self.hbase_host,
port=self.hbase_port,
autoconnect=True)
self.reconnect_count = 0
log.info('init hbase: {}'.format(host))
def de_initialize(self):
try:
log.info('try to de-initialize the system management.')
if not self.has_been_initializer:
log.info('base has not been initialized.')
return
self.has_been_initializer = False
log.info('base de initialize successfully.')
except Exception as err:
log.info('stop base initializer error {}'.format(err))
def initialize(self):
log.info('try to initialize the base wrappers.')
if self.has_been_initializer:
log.info('system management has been initialized yet.')
return
# Attention: Don't change the order for initializing component.
# env --> etcd --> setting --> redis --> stat
self._init_redis()
self._init_mysql()
self._init_hdfs()
self._init_hbase()
log.info('base initialize successfully.')
self.has_been_initializer = True
def create_tables(self, base):
try:
log.info('creating sql tables.')
base.metadata.create_all(self.engine)
except Exception as e:
log.error('create tables failed {}'.format(str(e)))
def __init__(self):
self.session = None
self.engine = None
log.info('init mysql: {}'.format(CONST.DB_URL))