def test_create_table(self): families = { 'cf1': dict(max_versions=10), 'cf2': dict(max_versions=1, block_cache_enabled=False), 'cf3': dict(), # use defaults } Globals.get_hbase_wrapper().create_table('test_table', families) table = Globals.get_hbase_wrapper().get_table('test_table') print(table)
def insert_material(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: ad_material = Material() ad_material.__dict__.update(dict_data) session.add(ad_material) return 'success'
def insert_resource(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: resource = Resources() resource.__dict__.update(dict_data) session.add(resource) return 'success'
def select_case(dict_data, page_no=1, page_size=10, sorted_way=-1): column_list = [ 'id', 'uid', 'type', 'update_time', 'ad_records', 'target_id', 'status', 'bidding' ] with Globals.get_mysql_wrapper().session_scope() as session: if sorted_way == -1: exce_query = session.query(AdCase).order_by( AdCase.update_time.desc()) else: exce_query = session.query(AdCase).order_by( AdCase.update_time.asc()) obj_list = exce_query.filter_by( status=dict_data['status'], asin=dict_data['asin']).offset(page_no - 1).limit(page_size) all_count = len(obj_list) result_dict = { 'all_count': all_count, 'cases': [{ key: obj.__dict__[key] for key in obj.__dict__ if key in column_list } for obj in obj_list] } return result_dict
def insert_material_record(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: material_record = MaterialRecord() material_record.__dict__.update(dict_data) session.add(material_record) mater_record_id = session.query(func.max(MaterialRecord.tid)).one()[0] return mater_record_id
def insert_case(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: ad_case = AdCase() ad_case.__dict__.update(dict_data) session.add(ad_case) new_id = session.query(func.max(AdCase.id)).one()[0] return new_id
def delete_resource(dict_data): # 查询resource_record表看是否有记录 with Globals.get_mysql_wrapper().session_scope as session: res_record = session.query(ResourceRecord).filter_by( depot_id=dict_data['id']).all() if len(res_record) > 0: # 有记录,返回提示信息 return 'failed to delete,this resource has been used in other ad_case' else: # 无记录,先查记录,再通过hdfs删除,再删除记录 with Globals.get_mysql_wrapper().session_scope() as session: target_obj = session.query(Resources).filter_by( id=dict_data['id']).first() if Globals.get_hdfs_wrapper.delete_hdfs(target_obj.addr): session.delete(target_obj) return 'success' else: return 'failed'
def insert_landing_page(dict_datas): ids = [] for data in dict_datas: with Globals.get_mysql_wrapper().session_scope() as session: landing_page = LandingPage() landing_page.__dict__.update(data) session.add(landing_page) lp_id = session.query(func.max(LandingPage.id)).one()[0] ids.append(lp_id) return ids
def select_material_record(): column_list = [ 'id', 'case_id', 'asin', 'resource', 'name', 'description', 'type', 'link' ] with Globals.get_mysql_wrapper().session_scope() as session: obj_list = session.query(MaterialRecord).all() result_dict = [{ key: obj.__dict__[key] for key in obj.__dict__ if key in column_list } for obj in obj_list] return result_dict
def select_landing_page_record(sorted_way=-1): column_list = [ 'id', 'case_id', 'landing_page_id', 'uid', 'template_id', 'attributes' ] with Globals.get_mysql_wrapper().session_scope() as session: exec_query = session.query(LandingPageRecord) if sorted_way == -1: obj_list = exec_query.order_by( LandingPageRecord.update_time.desc()).all() else: obj_list = exec_query.order_by( LandingPageRecord.update_time.asc()).all() result_dict = [{ key: obj.__dict__[key] for key in obj.__dict__ if key in column_list } for obj in obj_list] return result_dict
def select_resource(uid=None, asin=None, sorted_way=-1, key_word=None): column_list = [ 'id', 'uid', 'asin', 'update_time', 'type', 'fb_hash', 'keywords', 'addr' ] with Globals.get_mysql_wrapper().session_scope() as session: if sorted_way == -1: exce_query = session.query(Resources).order_by( Resources.update_time.desc()) else: exce_query = session.query(Resources).order_by( Resources.update_time.asc()) obj_list = exce_query.filter_by(uid=uid, asin=asin).filter( Resources.keywords.like("%{}%".format(key_word))).all() result_dict = [{ key: obj.__dict__[key] for key in obj.__dict__ if key in column_list } for obj in obj_list] return result_dict
def select_landing_page(data_dict, sorted_way=-1): column_list = [ 'id', 'uid', 'update_time', 'template_id', 'attributes', ] with Globals.get_mysql_wrapper().session_scope() as session: if sorted_way == -1: exec_query = session.query(LandingPage).order_by( LandingPage.update_time.desc()) else: exec_query = session.query(LandingPage).order_by( LandingPage.update_time.asc()) obj_list = exec_query.filter_by(uid=data_dict['uid'], id=data_dict['id']) result_dict = [{ key: obj.__dict__[key] for key in obj.__dict__ if key in column_list } for obj in obj_list] return result_dict
def select_material(asin, uid, sorted_way=-1, key_word=None): column_list = [ 'asin', 'uid', 'update_time', 'product_name', 'price', 'name', 'description', 'links', 'land_page_ids', 'keywords' ] with Globals.get_mysql_wrapper().session_scope() as session: if sorted_way == -1: exc_query = session.query(Material).order_by(Material.update_time.desc()) else: exc_query = session.query(Material).order_by(Material.update_time.asc()) obj_list = exc_query.filter_by(asin=asin, uid=uid).filter( Material.keywords.like("%{}%".format(key_word))).all() result_dict = [{key: obj.__dict__[key] for key in obj.__dict__ if key in column_list} for obj in obj_list] return result_dict
def del_case(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: session.query(AdCase).filter_by(uid=dict_data['uid']).filter( AdCase.id.in_(dict_data['id'])).delete(synchronize_session=False) return 'success'
def update_case_status(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: session.query(AdCase).filter_by(id=dict_data['id']).update( status=dict_data['status']) return 'success'
def update_material_record(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: session.query(MaterialRecord).filter_by( id=dict_data['id']).update(dict_data) return 'success'
def del_material_record(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: target_obj = session.query(MaterialRecord).filter_by( id=dict_data['id']).first() session.delete(target_obj) return 'success'
def test_connect(self): Globals.get_hbase_wrapper().connect()
def update_resource(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: session.query(Resources).filter_by( id=dict_data['id']).update(dict_data) return 'success'
def del_material(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: target_obj = session.query(Material).filter_by(asin=dict_data['asin']).first() session.delete(target_obj) return 'success'
def update_material(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: session.query(Material).filter_by(asin=dict_data['asin']).update(dict_data) return 'success'
def _init_hdfs(cls): hdfs_wrapper = HdfsWrapper() hdfs_wrapper.connect_hdfs() Globals.set_hdfs_wrapper(hdfs_wrapper)
def _init_hbase(cls): hbase_wrapper = HbaseWrapper(host=CONST.HBASE_HOST, port=CONST.HBASE_PORT) Globals.set_hbase_wrapper(hbase_wrapper)
def _init_mysql(cls): mysql_wrapper = MysqlWrapper() mysql_wrapper.connect_mysql(CONST.DB_NAME) mysql_wrapper.create_tables(Base) Globals.set_mysql_wrapper(mysql_wrapper)
def _init_redis(cls): redis_wrapper = RedisWrapper(host=CONST.REDIS_HOST, port=CONST.REDIS_PORT) Globals.set_redis_wrapper(redis_wrapper)
def del_landing_page(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: target_obj = session.query(LandingPage).filter_by( id=dict_data['id']).first() session.delete(target_obj) return 'success'
def update_landing_page(dict_data): with Globals.get_mysql_wrapper().session_scope() as session: session.query(LandingPage).filter_by( id=dict_data['id']).update(dict_data) return 'success'