def is_base_rpt_correct(cls, shop_info):
rpt_base_db = RptBaseDB(mongoConn, str(shop_info['sid']))
summary_rpt = rpt_base_db.get_one_day_summary_rpt(RptAll.YESTERDAY)
summary_rpt_stand = {}
for campaign_id in shop_info['campaign_ids']:
rpt_base_yesterday = SimbaRptCampaignBaseGet.get_yesterday_rpt_campaignbase_list(
shop_info['nick'], campaign_id, 'SEARCH,CAT', '1,2',
shop_info['access_token'], shop_info['subway_token'])
summary_rpt_stand[campaign_id] = {
'impressions': rpt_base_yesterday['impression'],
'click': rpt_base_yesterday['click'],
'cost': rpt_base_yesterday['cost']
}
for campaign_id in summary_rpt_stand.keys():
if summary_rpt_stand[campaign_id]['click'] == 0 and summary_rpt_stand[campaign_id]['cost'] == 0 \
and summary_rpt_stand[campaign_id]['impressions'] == 0:
continue
if not summary_rpt.has_key(campaign_id):
return False
if summary_rpt_stand[campaign_id] != summary_rpt[campaign_id]:
return False
return True
def is_base_rpt_correct(cls, shop_info):
rpt_base_db = RptBaseDB(mongoConn, str(shop_info["sid"]))
summary_rpt = rpt_base_db.get_one_day_summary_rpt(RptAll.YESTERDAY)
summary_rpt_stand = {}
for campaign_id in shop_info["campaign_ids"]:
rpt_base_yesterday = SimbaRptCampaignBaseGet.get_yesterday_rpt_campaignbase_list(
shop_info["nick"],
campaign_id,
"SEARCH,CAT",
"1,2",
shop_info["access_token"],
shop_info["subway_token"],
)
summary_rpt_stand[campaign_id] = {
"impressions": rpt_base_yesterday["impression"],
"click": rpt_base_yesterday["click"],
"cost": rpt_base_yesterday["cost"],
}
for campaign_id in summary_rpt_stand.keys():
if (
summary_rpt_stand[campaign_id]["click"] == 0
and summary_rpt_stand[campaign_id]["cost"] == 0
and summary_rpt_stand[campaign_id]["impressions"] == 0
):
continue
if not summary_rpt.has_key(campaign_id):
return False
if summary_rpt_stand[campaign_id] != summary_rpt[campaign_id]:
return False
return True
def get_base_data(cls, shop_info_list):
for shop_info in shop_info_list:
try:
if RptAll.is_base_rpt_correct(shop_info):
continue
report_logger.info("店铺 [%s] 需要重新获取基础报表", shop_info['nick'])
rpt_base_db = RptBaseDB(mongoConn, str(shop_info['sid']))
rpt_base_db.clear_one_day_ago_rpt(RptAll.ONE_WEEK_AGO)
rpt_base_db.clear_one_day_rpt(RptAll.YESTERDAY)
base_rpt_get = BaseRptGet(shop_info['nick'],
shop_info['access_token'],
shop_info['subway_token'],
shop_info['campaign_ids'])
base_rpt_list = base_rpt_get.yesterday_rpt_get()
rpt_base_db.save_records(base_rpt_list)
report_logger.info("店铺 [%s] 重新获取基础报表完成，条数 %d",
shop_info['nick'], len(base_rpt_list))
except Exception, data:
report_logger.exception('get base data exception')
def get_base_data(cls, shop_info_list):
for shop_info in shop_info_list:
try:
if RptAll.is_base_rpt_correct(shop_info):
continue
report_logger.info("店铺 [%s] 需要重新获取基础报表", shop_info["nick"])
rpt_base_db = RptBaseDB(mongoConn, str(shop_info["sid"]))
rpt_base_db.clear_one_day_ago_rpt(RptAll.ONE_WEEK_AGO)
rpt_base_db.clear_one_day_rpt(RptAll.YESTERDAY)
base_rpt_get = BaseRptGet(
shop_info["nick"], shop_info["access_token"], shop_info["subway_token"], shop_info["campaign_ids"]
)
base_rpt_list = base_rpt_get.yesterday_rpt_get()
rpt_base_db.save_records(base_rpt_list)
report_logger.info("店铺 [%s] 重新获取基础报表完成，条数 %d", shop_info["nick"], len(base_rpt_list))
except Exception, data:
report_logger.exception("get base data exception")
class RptAll(object):
"""
根据店铺信息列表 获取所有的报表信息
"""
YESTERDAY = ''
ONE_WEEK_AGO = ''
@classmethod
def get_day_info(cls):
RptAll.YESTERDAY = DateHandle.date_to_string(
DateHandle.get_yesterday_date())
RptAll.ONE_WEEK_AGO = DateHandle.date_to_string(
DateHandle.get_one_week_ago_date())
@classmethod
def get_base_data_topats(cls, shop_info_list):
"""
获取店铺的所有计划的base数据
"""
#初始化异步报表下载
fail_shop_info_list = []
rpt_topats_list_mid = []
shop_info_list_mid = []
for shop_info in shop_info_list:
try:
campaign_ids = shop_info['campaign_ids']
base_rpt_topats = BaseRptTopatsGet(shop_info['sid'],
shop_info['nick'],
shop_info['access_token'],
shop_info['subway_token'],
campaign_ids)
base_rpt_topats.generate_base_task_id()
except Exception, data:
report_logger.exception('初始化异步报表出错')
fail_shop_info_list.append(shop_info)
continue
rpt_topats_list_mid.append(base_rpt_topats)
shop_info_list_mid.append(shop_info)
time.sleep(2)
time.sleep(TOPATS_SLEEP_TIME)
#time.sleep(20)
#获取下载链接, 下载报表并存储
for i in range(0, len(shop_info_list_mid)):
rpt_topats = rpt_topats_list_mid[i]
try:
if not rpt_topats.check_task_ok():
fail_shop_info_list.append(shop_info_list_mid[i])
if not rpt_topats.download_rpt():
fail_shop_info_list.append(shop_info_list_mid[i])
if not rpt_topats.parse_rpt():
fail_shop_info_list.append(shop_info_list_mid[i])
rpt_base_db = RptBaseDB(mongoConn,
str(shop_info_list_mid[i]['sid']))
rpt_base_db.clear_one_day_ago_rpt(RptAll.ONE_WEEK_AGO)
rpt_base_db.clear_one_day_rpt(RptAll.YESTERDAY)
rpt_base_db.save_records(rpt_topats.rpt_record_list)
except Exception, data:
report_logger.exception('获取下载链接，抓取或存储出错')
fail_shop_info_list.append(shop_info_list_mid[i])
time.sleep(0.2)
