def get_json_data_by_resp(resp):
try:
data_dict = json_loads(resp.content.decode(CONST.UTF_8))
except Exception as err:
log.error(str(err))
return None
return data_dict
def read_hdfs(self, hdfs_path):
try:
with self.client.read(hdfs_path) as reader:
return reader.read()
except:
log.error(traceback.format_exc())
self.connect_hdfs()
log.error('reconnect hdfs...')
def write_hdfs(cls, path, value, overwrite=True):
try:
with cls._client.write(path, overwrite=overwrite) as writer:
writer.write(value)
except:
log.error(traceback.format_exc())
cls.connect_hdfs()
log.error('reconnect hdfs...')
def read_hdfs(cls, path):
try:
with cls._client.read(path) as reader:
return reader.read()
except:
log.error(traceback.format_exc())
cls.connect_hdfs()
log.error('reconnect hdfs...')
def write_hdfs(self, hdfs_path, data, overwrite=False):
try:
with self.client.write(hdfs_path, overwrite=overwrite) as writer:
writer.write(data)
return hdfs_path
except:
log.error(traceback.format_exc())
self.connect_hdfs()
log.error('reconnect hdfs...')
def del_table(self, table_name):
try:
with self.connection_pool.connection(self.timeout) as connection:
connection.delete_table(table_name)
log.debug('delete table: {} successfully.'.format(table_name))
except Exception as e:
log.error('delete table: {} get fail.'.format(table_name, str(e)))
raise Exception(str(e))
def get_table(self, table_name):
try:
with self.connection_pool.connection(self.timeout) as connection:
target_table = connection.table(table_name)
log.debug('target table: {} get successfully.'.format(table_name))
except Exception as e:
log.error('target table: {} get fail.'.format(table_name, str(e)))
raise Exception(str(e))
return target_table
def create_table(self, table_name, families):
try:
with self.connection_pool.connection(self.timeout) as connection:
connection.create_table(table_name, families)
log.info(
'target table: {} create successfully.'.format(table_name))
except Exception as e:
log.error('target table: {} create fail.'.format(
table_name, str(e)))
raise Exception(str(e))
def session_scope(self):
try:
# with 代码执行的部分
yield self.session
# with包裹的代码
self.session.commit()
except Exception:
self.session.rollback()
log.error(traceback.format_exc(30))
finally:
self.session.close()
def connect(self):
try:
self.connection_pool.connection(self.timeout)
except Exception as e:
while self.reconnect_count < CONST.HAPPYBASE_RECONNECT_TIMES:
self.reconnect_count += 1
log.error(
'connect hbase: {} error,start reconnecting,has retried {} times'
.format(self.hbase_host, self.reconnect_count))
self.connect()
time.sleep(CONST.HAPPYBASE_RECONNECT_WAIT_SECONDS)
log.error('reconnect times have been max,please check the error')
raise Exception(str(e))
def get(self, block=False, timeout=None):
try:
if block:
item = self.redis_conn.blpop(self.key, timeout=timeout)
else:
item = self.redis_conn.lpop(self.key)
if item:
item_result = item.decode(CONST.UTF_8)
else:
item_result = None
except Exception as error:
log.error(str(error))
item_result = None
return item_result
def connect_mysql(self, db_name):
try:
self.engine = create_engine(
'mysql+mysqlconnector://{username}:{pwd}@{db_url}/{db_name}?charset=utf8'
.format(username=CONST.MYSQL_USERNAME,
pwd=CONST.MYSQL_PWD,
db_url=CONST.DB_URL,
db_name=db_name),
max_overflow=5,
encoding='utf-8')
DB_Session = sessionmaker(bind=self.engine)
self.session = DB_Session()
except Exception as e:
log.error('{}'.format('', str(e)))
raise traceback.format_exc(30)
def put(self):
try:
json_data = request.get_json(force=True)
case_data = json_data['case']
lp_record_data = json_data['landing_page_record']
mater_record_data = json_data['material_record']
uid = case_data['uid']
except KeyError as e:
log.error(traceback.format_exc())
return {
'error_code': 56003,
'reason': 'the key is error or is not exits',
'result': 'failure'
}
except Exception as e:
log.error(traceback.format_exc())
return {'reason': str(e), 'result': 'failure'}
message = update_case(json_data)
return {'result': message}
def post(self):
json_data = request.get_json(force=True)
try:
case_data = json_data['case']
lp_record_data = json_data['landing_page_record']
mater_record_data = json_data['material_record']
uid = case_data['uid']
except KeyError as e:
log.error(traceback.format_exc())
return {
'error_code': 56003,
'reason': 'the key is error or is not exits',
'result': 'failure'
}
except Exception as e:
log.error(traceback.format_exc())
return {'reason': str(e), 'result': 'failure'}
case_id = insert_case(case_data)
# 如果is_save为true，存落地页库，再存记录；否则只存记录
if lp_record_data['is_save'] == 'true':
for lp_record in lp_record_data:
lp_ids = insert_landing_page(lp_record)
lp_record['landing_page_id'] = lp_ids[0]
lp_record['case_id'] = case_id
lp_record_id = insert_landing_page_record(lp_record)
for lp_record in lp_record_data:
lp_record['case_id'] = case_id
lp_record_ids = insert_landing_page_record(lp_record_data)
for mater_record in mater_record_data:
res_record = {}
res_record['case_id'] = case_id
res_record['resource_id'] = mater_record['id']
res_record['uid'] = uid
res_record_id = insert_resource_record(res_record)
mater_record['resource'] = res_record_id
mater_record['case_id'] = case_id
mater_record_id = insert_material_record(mater_record)
return {'result': 'success'}
def create_tables(self, base):
try:
log.info('creating sql tables.')
base.metadata.create_all(self.engine)
except Exception as e:
log.error('create tables failed {}'.format(str(e)))
def init():
try:
BS_INITIALIZER.initialize()
except Exception as e:
log.error('{}'.format(str(e)))
raise e