def get_json_data_by_resp(resp): try: data_dict = json_loads(resp.content.decode(CONST.UTF_8)) except Exception as err: log.error(str(err)) return None return data_dict
def read_hdfs(self, hdfs_path): try: with self.client.read(hdfs_path) as reader: return reader.read() except: log.error(traceback.format_exc()) self.connect_hdfs() log.error('reconnect hdfs...')
def write_hdfs(cls, path, value, overwrite=True): try: with cls._client.write(path, overwrite=overwrite) as writer: writer.write(value) except: log.error(traceback.format_exc()) cls.connect_hdfs() log.error('reconnect hdfs...')
def read_hdfs(cls, path): try: with cls._client.read(path) as reader: return reader.read() except: log.error(traceback.format_exc()) cls.connect_hdfs() log.error('reconnect hdfs...')
def write_hdfs(self, hdfs_path, data, overwrite=False): try: with self.client.write(hdfs_path, overwrite=overwrite) as writer: writer.write(data) return hdfs_path except: log.error(traceback.format_exc()) self.connect_hdfs() log.error('reconnect hdfs...')
def del_table(self, table_name): try: with self.connection_pool.connection(self.timeout) as connection: connection.delete_table(table_name) log.debug('delete table: {} successfully.'.format(table_name)) except Exception as e: log.error('delete table: {} get fail.'.format(table_name, str(e))) raise Exception(str(e))
def get_table(self, table_name): try: with self.connection_pool.connection(self.timeout) as connection: target_table = connection.table(table_name) log.debug('target table: {} get successfully.'.format(table_name)) except Exception as e: log.error('target table: {} get fail.'.format(table_name, str(e))) raise Exception(str(e)) return target_table
def create_table(self, table_name, families): try: with self.connection_pool.connection(self.timeout) as connection: connection.create_table(table_name, families) log.info( 'target table: {} create successfully.'.format(table_name)) except Exception as e: log.error('target table: {} create fail.'.format( table_name, str(e))) raise Exception(str(e))
def session_scope(self): try: # with 代码执行的部分 yield self.session # with包裹的代码 self.session.commit() except Exception: self.session.rollback() log.error(traceback.format_exc(30)) finally: self.session.close()
def connect(self): try: self.connection_pool.connection(self.timeout) except Exception as e: while self.reconnect_count < CONST.HAPPYBASE_RECONNECT_TIMES: self.reconnect_count += 1 log.error( 'connect hbase: {} error,start reconnecting,has retried {} times' .format(self.hbase_host, self.reconnect_count)) self.connect() time.sleep(CONST.HAPPYBASE_RECONNECT_WAIT_SECONDS) log.error('reconnect times have been max,please check the error') raise Exception(str(e))
def get(self, block=False, timeout=None): try: if block: item = self.redis_conn.blpop(self.key, timeout=timeout) else: item = self.redis_conn.lpop(self.key) if item: item_result = item.decode(CONST.UTF_8) else: item_result = None except Exception as error: log.error(str(error)) item_result = None return item_result
def connect_mysql(self, db_name): try: self.engine = create_engine( 'mysql+mysqlconnector://{username}:{pwd}@{db_url}/{db_name}?charset=utf8' .format(username=CONST.MYSQL_USERNAME, pwd=CONST.MYSQL_PWD, db_url=CONST.DB_URL, db_name=db_name), max_overflow=5, encoding='utf-8') DB_Session = sessionmaker(bind=self.engine) self.session = DB_Session() except Exception as e: log.error('{}'.format('', str(e))) raise traceback.format_exc(30)
def put(self): try: json_data = request.get_json(force=True) case_data = json_data['case'] lp_record_data = json_data['landing_page_record'] mater_record_data = json_data['material_record'] uid = case_data['uid'] except KeyError as e: log.error(traceback.format_exc()) return { 'error_code': 56003, 'reason': 'the key is error or is not exits', 'result': 'failure' } except Exception as e: log.error(traceback.format_exc()) return {'reason': str(e), 'result': 'failure'} message = update_case(json_data) return {'result': message}
def post(self): json_data = request.get_json(force=True) try: case_data = json_data['case'] lp_record_data = json_data['landing_page_record'] mater_record_data = json_data['material_record'] uid = case_data['uid'] except KeyError as e: log.error(traceback.format_exc()) return { 'error_code': 56003, 'reason': 'the key is error or is not exits', 'result': 'failure' } except Exception as e: log.error(traceback.format_exc()) return {'reason': str(e), 'result': 'failure'} case_id = insert_case(case_data) # 如果is_save为true,存落地页库,再存记录;否则只存记录 if lp_record_data['is_save'] == 'true': for lp_record in lp_record_data: lp_ids = insert_landing_page(lp_record) lp_record['landing_page_id'] = lp_ids[0] lp_record['case_id'] = case_id lp_record_id = insert_landing_page_record(lp_record) for lp_record in lp_record_data: lp_record['case_id'] = case_id lp_record_ids = insert_landing_page_record(lp_record_data) for mater_record in mater_record_data: res_record = {} res_record['case_id'] = case_id res_record['resource_id'] = mater_record['id'] res_record['uid'] = uid res_record_id = insert_resource_record(res_record) mater_record['resource'] = res_record_id mater_record['case_id'] = case_id mater_record_id = insert_material_record(mater_record) return {'result': 'success'}
def create_tables(self, base): try: log.info('creating sql tables.') base.metadata.create_all(self.engine) except Exception as e: log.error('create tables failed {}'.format(str(e)))
def init(): try: BS_INITIALIZER.initialize() except Exception as e: log.error('{}'.format(str(e))) raise e