def test_create_table(self):
families = {
'cf1': dict(max_versions=10),
'cf2': dict(max_versions=1, block_cache_enabled=False),
'cf3': dict(), # use defaults
}
Globals.get_hbase_wrapper().create_table('test_table', families)
table = Globals.get_hbase_wrapper().get_table('test_table')
print(table)
def insert_material(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
ad_material = Material()
ad_material.__dict__.update(dict_data)
session.add(ad_material)
return 'success'
def insert_resource(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
resource = Resources()
resource.__dict__.update(dict_data)
session.add(resource)
return 'success'
def select_case(dict_data, page_no=1, page_size=10, sorted_way=-1):
column_list = [
'id', 'uid', 'type', 'update_time', 'ad_records', 'target_id',
'status', 'bidding'
]
with Globals.get_mysql_wrapper().session_scope() as session:
if sorted_way == -1:
exce_query = session.query(AdCase).order_by(
AdCase.update_time.desc())
else:
exce_query = session.query(AdCase).order_by(
AdCase.update_time.asc())
obj_list = exce_query.filter_by(
status=dict_data['status'],
asin=dict_data['asin']).offset(page_no - 1).limit(page_size)
all_count = len(obj_list)
result_dict = {
'all_count':
all_count,
'cases': [{
key: obj.__dict__[key]
for key in obj.__dict__ if key in column_list
} for obj in obj_list]
}
return result_dict
def insert_material_record(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
material_record = MaterialRecord()
material_record.__dict__.update(dict_data)
session.add(material_record)
mater_record_id = session.query(func.max(MaterialRecord.tid)).one()[0]
return mater_record_id
def insert_case(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
ad_case = AdCase()
ad_case.__dict__.update(dict_data)
session.add(ad_case)
new_id = session.query(func.max(AdCase.id)).one()[0]
return new_id
def delete_resource(dict_data):
# 查询resource_record表看是否有记录
with Globals.get_mysql_wrapper().session_scope as session:
res_record = session.query(ResourceRecord).filter_by(
depot_id=dict_data['id']).all()
if len(res_record) > 0:
# 有记录，返回提示信息
return 'failed to delete,this resource has been used in other ad_case'
else:
# 无记录，先查记录,再通过hdfs删除，再删除记录
with Globals.get_mysql_wrapper().session_scope() as session:
target_obj = session.query(Resources).filter_by(
id=dict_data['id']).first()
if Globals.get_hdfs_wrapper.delete_hdfs(target_obj.addr):
session.delete(target_obj)
return 'success'
else:
return 'failed'
def insert_landing_page(dict_datas):
ids = []
for data in dict_datas:
with Globals.get_mysql_wrapper().session_scope() as session:
landing_page = LandingPage()
landing_page.__dict__.update(data)
session.add(landing_page)
lp_id = session.query(func.max(LandingPage.id)).one()[0]
ids.append(lp_id)
return ids
def select_material_record():
column_list = [
'id', 'case_id', 'asin', 'resource', 'name', 'description', 'type',
'link'
]
with Globals.get_mysql_wrapper().session_scope() as session:
obj_list = session.query(MaterialRecord).all()
result_dict = [{
key: obj.__dict__[key]
for key in obj.__dict__ if key in column_list
} for obj in obj_list]
return result_dict
def select_landing_page_record(sorted_way=-1):
column_list = [
'id', 'case_id', 'landing_page_id', 'uid', 'template_id', 'attributes'
]
with Globals.get_mysql_wrapper().session_scope() as session:
exec_query = session.query(LandingPageRecord)
if sorted_way == -1:
obj_list = exec_query.order_by(
LandingPageRecord.update_time.desc()).all()
else:
obj_list = exec_query.order_by(
LandingPageRecord.update_time.asc()).all()
result_dict = [{
key: obj.__dict__[key]
for key in obj.__dict__ if key in column_list
} for obj in obj_list]
return result_dict
def select_resource(uid=None, asin=None, sorted_way=-1, key_word=None):
column_list = [
'id', 'uid', 'asin', 'update_time', 'type', 'fb_hash', 'keywords',
'addr'
]
with Globals.get_mysql_wrapper().session_scope() as session:
if sorted_way == -1:
exce_query = session.query(Resources).order_by(
Resources.update_time.desc())
else:
exce_query = session.query(Resources).order_by(
Resources.update_time.asc())
obj_list = exce_query.filter_by(uid=uid, asin=asin).filter(
Resources.keywords.like("%{}%".format(key_word))).all()
result_dict = [{
key: obj.__dict__[key]
for key in obj.__dict__ if key in column_list
} for obj in obj_list]
return result_dict
def select_landing_page(data_dict, sorted_way=-1):
column_list = [
'id',
'uid',
'update_time',
'template_id',
'attributes',
]
with Globals.get_mysql_wrapper().session_scope() as session:
if sorted_way == -1:
exec_query = session.query(LandingPage).order_by(
LandingPage.update_time.desc())
else:
exec_query = session.query(LandingPage).order_by(
LandingPage.update_time.asc())
obj_list = exec_query.filter_by(uid=data_dict['uid'],
id=data_dict['id'])
result_dict = [{
key: obj.__dict__[key]
for key in obj.__dict__ if key in column_list
} for obj in obj_list]
return result_dict
def select_material(asin, uid, sorted_way=-1, key_word=None):
column_list = [
'asin',
'uid',
'update_time',
'product_name',
'price',
'name',
'description',
'links',
'land_page_ids',
'keywords'
]
with Globals.get_mysql_wrapper().session_scope() as session:
if sorted_way == -1:
exc_query = session.query(Material).order_by(Material.update_time.desc())
else:
exc_query = session.query(Material).order_by(Material.update_time.asc())
obj_list = exc_query.filter_by(asin=asin, uid=uid).filter(
Material.keywords.like("%{}%".format(key_word))).all()
result_dict = [{key: obj.__dict__[key] for key in obj.__dict__ if key in column_list} for obj in obj_list]
return result_dict
def del_case(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
session.query(AdCase).filter_by(uid=dict_data['uid']).filter(
AdCase.id.in_(dict_data['id'])).delete(synchronize_session=False)
return 'success'
def update_case_status(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
session.query(AdCase).filter_by(id=dict_data['id']).update(
status=dict_data['status'])
return 'success'
def update_material_record(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
session.query(MaterialRecord).filter_by(
id=dict_data['id']).update(dict_data)
return 'success'
def del_material_record(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
target_obj = session.query(MaterialRecord).filter_by(
id=dict_data['id']).first()
session.delete(target_obj)
return 'success'
def test_connect(self):
Globals.get_hbase_wrapper().connect()
def update_resource(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
session.query(Resources).filter_by(
id=dict_data['id']).update(dict_data)
return 'success'
def del_material(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
target_obj = session.query(Material).filter_by(asin=dict_data['asin']).first()
session.delete(target_obj)
return 'success'
def update_material(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
session.query(Material).filter_by(asin=dict_data['asin']).update(dict_data)
return 'success'
def _init_hdfs(cls):
hdfs_wrapper = HdfsWrapper()
hdfs_wrapper.connect_hdfs()
Globals.set_hdfs_wrapper(hdfs_wrapper)
def _init_hbase(cls):
hbase_wrapper = HbaseWrapper(host=CONST.HBASE_HOST,
port=CONST.HBASE_PORT)
Globals.set_hbase_wrapper(hbase_wrapper)
def _init_mysql(cls):
mysql_wrapper = MysqlWrapper()
mysql_wrapper.connect_mysql(CONST.DB_NAME)
mysql_wrapper.create_tables(Base)
Globals.set_mysql_wrapper(mysql_wrapper)
def _init_redis(cls):
redis_wrapper = RedisWrapper(host=CONST.REDIS_HOST,
port=CONST.REDIS_PORT)
Globals.set_redis_wrapper(redis_wrapper)
def del_landing_page(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
target_obj = session.query(LandingPage).filter_by(
id=dict_data['id']).first()
session.delete(target_obj)
return 'success'
def update_landing_page(dict_data):
with Globals.get_mysql_wrapper().session_scope() as session:
session.query(LandingPage).filter_by(
id=dict_data['id']).update(dict_data)
return 'success'
