def is_base_rpt_correct(cls, shop_info): rpt_base_db = RptBaseDB(mongoConn, str(shop_info['sid'])) summary_rpt = rpt_base_db.get_one_day_summary_rpt(RptAll.YESTERDAY) summary_rpt_stand = {} for campaign_id in shop_info['campaign_ids']: rpt_base_yesterday = SimbaRptCampaignBaseGet.get_yesterday_rpt_campaignbase_list( shop_info['nick'], campaign_id, 'SEARCH,CAT', '1,2', shop_info['access_token'], shop_info['subway_token']) summary_rpt_stand[campaign_id] = { 'impressions': rpt_base_yesterday['impression'], 'click': rpt_base_yesterday['click'], 'cost': rpt_base_yesterday['cost'] } for campaign_id in summary_rpt_stand.keys(): if summary_rpt_stand[campaign_id]['click'] == 0 and summary_rpt_stand[campaign_id]['cost'] == 0 \ and summary_rpt_stand[campaign_id]['impressions'] == 0: continue if not summary_rpt.has_key(campaign_id): return False if summary_rpt_stand[campaign_id] != summary_rpt[campaign_id]: return False return True
def is_base_rpt_correct(cls, shop_info): rpt_base_db = RptBaseDB(mongoConn, str(shop_info["sid"])) summary_rpt = rpt_base_db.get_one_day_summary_rpt(RptAll.YESTERDAY) summary_rpt_stand = {} for campaign_id in shop_info["campaign_ids"]: rpt_base_yesterday = SimbaRptCampaignBaseGet.get_yesterday_rpt_campaignbase_list( shop_info["nick"], campaign_id, "SEARCH,CAT", "1,2", shop_info["access_token"], shop_info["subway_token"], ) summary_rpt_stand[campaign_id] = { "impressions": rpt_base_yesterday["impression"], "click": rpt_base_yesterday["click"], "cost": rpt_base_yesterday["cost"], } for campaign_id in summary_rpt_stand.keys(): if ( summary_rpt_stand[campaign_id]["click"] == 0 and summary_rpt_stand[campaign_id]["cost"] == 0 and summary_rpt_stand[campaign_id]["impressions"] == 0 ): continue if not summary_rpt.has_key(campaign_id): return False if summary_rpt_stand[campaign_id] != summary_rpt[campaign_id]: return False return True
def get_base_data(cls, shop_info_list): for shop_info in shop_info_list: try: if RptAll.is_base_rpt_correct(shop_info): continue report_logger.info("店铺 [%s] 需要重新获取基础报表", shop_info['nick']) rpt_base_db = RptBaseDB(mongoConn, str(shop_info['sid'])) rpt_base_db.clear_one_day_ago_rpt(RptAll.ONE_WEEK_AGO) rpt_base_db.clear_one_day_rpt(RptAll.YESTERDAY) base_rpt_get = BaseRptGet(shop_info['nick'], shop_info['access_token'], shop_info['subway_token'], shop_info['campaign_ids']) base_rpt_list = base_rpt_get.yesterday_rpt_get() rpt_base_db.save_records(base_rpt_list) report_logger.info("店铺 [%s] 重新获取基础报表完成,条数 %d", shop_info['nick'], len(base_rpt_list)) except Exception, data: report_logger.exception('get base data exception')
def get_base_data(cls, shop_info_list): for shop_info in shop_info_list: try: if RptAll.is_base_rpt_correct(shop_info): continue report_logger.info("店铺 [%s] 需要重新获取基础报表", shop_info["nick"]) rpt_base_db = RptBaseDB(mongoConn, str(shop_info["sid"])) rpt_base_db.clear_one_day_ago_rpt(RptAll.ONE_WEEK_AGO) rpt_base_db.clear_one_day_rpt(RptAll.YESTERDAY) base_rpt_get = BaseRptGet( shop_info["nick"], shop_info["access_token"], shop_info["subway_token"], shop_info["campaign_ids"] ) base_rpt_list = base_rpt_get.yesterday_rpt_get() rpt_base_db.save_records(base_rpt_list) report_logger.info("店铺 [%s] 重新获取基础报表完成,条数 %d", shop_info["nick"], len(base_rpt_list)) except Exception, data: report_logger.exception("get base data exception")
class RptAll(object): """ 根据店铺信息列表 获取所有的报表信息 """ YESTERDAY = '' ONE_WEEK_AGO = '' @classmethod def get_day_info(cls): RptAll.YESTERDAY = DateHandle.date_to_string( DateHandle.get_yesterday_date()) RptAll.ONE_WEEK_AGO = DateHandle.date_to_string( DateHandle.get_one_week_ago_date()) @classmethod def get_base_data_topats(cls, shop_info_list): """ 获取店铺的所有计划的base数据 """ #初始化异步报表下载 fail_shop_info_list = [] rpt_topats_list_mid = [] shop_info_list_mid = [] for shop_info in shop_info_list: try: campaign_ids = shop_info['campaign_ids'] base_rpt_topats = BaseRptTopatsGet(shop_info['sid'], shop_info['nick'], shop_info['access_token'], shop_info['subway_token'], campaign_ids) base_rpt_topats.generate_base_task_id() except Exception, data: report_logger.exception('初始化异步报表出错') fail_shop_info_list.append(shop_info) continue rpt_topats_list_mid.append(base_rpt_topats) shop_info_list_mid.append(shop_info) time.sleep(2) time.sleep(TOPATS_SLEEP_TIME) #time.sleep(20) #获取下载链接, 下载报表并存储 for i in range(0, len(shop_info_list_mid)): rpt_topats = rpt_topats_list_mid[i] try: if not rpt_topats.check_task_ok(): fail_shop_info_list.append(shop_info_list_mid[i]) if not rpt_topats.download_rpt(): fail_shop_info_list.append(shop_info_list_mid[i]) if not rpt_topats.parse_rpt(): fail_shop_info_list.append(shop_info_list_mid[i]) rpt_base_db = RptBaseDB(mongoConn, str(shop_info_list_mid[i]['sid'])) rpt_base_db.clear_one_day_ago_rpt(RptAll.ONE_WEEK_AGO) rpt_base_db.clear_one_day_rpt(RptAll.YESTERDAY) rpt_base_db.save_records(rpt_topats.rpt_record_list) except Exception, data: report_logger.exception('获取下载链接,抓取或存储出错') fail_shop_info_list.append(shop_info_list_mid[i]) time.sleep(0.2)