def create_languages_for_all_campaigns_dataset(date_range): try: with open( f'{os.environ.get("ULANMEDIAAPP")}/data/complete_languages/{date_range}_complete_languages_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] data = json_file["data"] languages_for_all_campaigns = {"metadata": metadata, "data": []} for language_name in data: languages_for_all_campaigns["data"].append( data[language_name]["for_all_campaigns"]) with open( f"{os.environ.get('ULANMEDIAAPP')}/data/languages_for_all_campaigns/{date_range}_languages_for_all_campaigns_dataset.json", "w") as file: json.dump(languages_for_all_campaigns, file) return json.dumps(languages_for_all_campaigns) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_languages_for_all_campaigns_dataset()", "Failed - create_languages_for_all_campaigns_dataset()") sys.exit()
def create_campaigns_for_one_ad_dataset(ad_image, date_range): try: with open( f'{os.environ.get("ULANMEDIAAPP")}/data/complete_ads/{date_range}_complete_ads_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] data = json_file["data"] campaigns_for_one_ad = { "metadata": metadata, "data": data[ad_image]["for_each_campaign"] } with open( f"{os.environ.get('ULANMEDIAAPP')}/data/campaigns_for_one_ad/{ad_image}_{date_range}_campaigns_for_one_ad_dataset.json", "w") as file: json.dump(campaigns_for_one_ad, file) return json.dumps(campaigns_for_one_ad) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_campaigns_for_one_ad_dataset()", "Failed - create_campaigns_for_one_ad_dataset()") sys.exit()
def create_p_widgets_for_one_domain_for_all_campaigns_dataset(date_range, domain): try: domains_to_check = domain.split(",") p_widgets_for_one_domain_for_all_campaigns = {"metadata": {"vol_start_date": "none", "vol_end_date": "none"}, "data": {}} with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_p_widgets/{date_range}_complete_p_widgets_dataset.json', 'r') as file: complete_p_widgets = json.load(file) for p_widget in complete_p_widgets: domains_to_match = complete_p_widgets[p_widget]["for_all_campaigns"]["domain"].split(",") for domain_to_check in domains_to_check: if domain_to_check in domains_to_match: p_widgets_for_one_domain_for_all_campaigns["data"][p_widget] = complete_p_widgets[p_widget]["for_all_campaigns"] p_widgets_for_one_domain_for_all_campaigns["data"][p_widget]["good_campaigns_count"] = complete_p_widgets[p_widget]["good_campaigns_count"] p_widgets_for_one_domain_for_all_campaigns["data"][p_widget]["bad_campaigns_count"] = complete_p_widgets[p_widget]["bad_campaigns_count"] p_widgets_for_one_domain_for_all_campaigns["data"][p_widget]["wait_campaigns_count"] = complete_p_widgets[p_widget]["wait_campaigns_count"] break if len(domain) > 20: domain = domain[:20] with open(f"{os.environ.get('ULANMEDIAAPP')}/data/p_widgets_for_one_domain_for_all_campaigns/{date_range}_{domain}_p_widgets_for_one_domain_for_all_campaigns_dataset.json", "w") as file: json.dump(p_widgets_for_one_domain_for_all_campaigns, file) return json.dumps(p_widgets_for_one_domain_for_all_campaigns) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_p_widgets_for_one_domain_for_all_campaigns_dataset()", "Failed - create_p_widgets_for_one_domain_for_all_campaigns_dataset()") sys.exit()
def create_campaigns_for_one_country_dataset(date_range, country_name): try: with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_countries/{date_range}_complete_countries_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] data = json_file["data"] campaigns_for_one_country = {"metadata": metadata, "data": data[country_name]["for_each_campaign"]} campaigns_sets = get_campaign_sets() campaigns_lookup = {} for campaign in campaigns_sets: campaigns_lookup[campaign["vol_id"]] = campaign["name"] for campaign_id in campaigns_for_one_country["data"]: campaigns_for_one_country["data"][campaign_id]["campaign_name"] = campaigns_lookup[campaign_id] with open(f"{os.environ.get('ULANMEDIAAPP')}/data/campaigns_for_one_country/{date_range}_{country_name}_campaigns_for_one_country_dataset.json", "w") as file: json.dump(campaigns_for_one_country, file) return json.dumps(campaigns_for_one_country) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_campaigns_for_one_country_dataset()", "Failed - create_campaigns_for_one_country_dataset()") sys.exit()
def update_blacklist_from_mikes_server(): try: res = requests.get( "https://ulanmedia.brianhaller.net/api/readblacklist") res.raise_for_status() widgets = res.json() mydb = mysql.connector.connect(host="localhost", user=mysql_user, passwd=mysql_password, database="ulanmedia") mycursor = mydb.cursor() for widget in widgets: sql = f"INSERT INTO colorlist(widget_id, color) values('{widget}', 'black');" mycursor.execute(sql) mydb.commit() except requests.exceptions.RequestException as e: print("Failed to update blacklist") print(e) send_email( "*****@*****.**", "Failed - update_backlist() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_mgid_access_token(login, password): try: res = requests.post("https://api.mgid.com/v1/auth/token", headers={ "Content-type": "application/x-www-form-urlencoded", "Cache-Control": "no-cache" }, data={ "email": login, "password": password }) try: token = res.json()["token"] return token except: send_email( "*****@*****.**", "Failed - mgid token access at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), "Http status code error - mgid token access failed.") sys.exit() except requests.exceptions.RequestException as e: print("Failed - get_mgid_access_token()") send_email( "*****@*****.**", "Failed - get_mgid_access_token() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def create_countries_for_one_campaign_dataset(date_range, vol_id): try: with open( f'{os.environ.get("ULANMEDIAAPP")}/data/complete_countries/{date_range}_complete_countries_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] data = json_file["data"] countries_for_one_campaign = {"metadata": metadata, "data": []} for country in data: for campaign in data[country]["for_each_campaign"]: if data[country]["for_each_campaign"][campaign][ "campaign_id"] == vol_id: countries_for_one_campaign["data"].append( data[country]["for_each_campaign"][campaign]) with open( f"{os.environ.get('ULANMEDIAAPP')}/data/countries_for_one_campaign/{date_range}_{vol_id}_countries_for_one_campaign_dataset.json", "w") as file: json.dump(countries_for_one_campaign, file) return json.dumps(countries_for_one_campaign) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_countries_for_one_campaign_dataset()", "Failed - create_countries_for_one_campaign_dataset()") sys.exit()
def create_ads_for_one_campaign_dataset(vol_id, date_range): try: with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_ads/{date_range}_complete_ads_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] data = json_file["data"] ads_for_one_campaign = {"metadata": metadata, "data": []} for ad_image in data.values(): for ad in ad_image["for_each_campaign"]: if vol_id == ad["vol_id"]: ads_for_one_campaign["data"].append(ad) # add the parent ad global rank for ad in ads_for_one_campaign["data"]: ad_image = ad["image"] ad["global_rank"] = data[ad_image]["for_all_campaigns"]["global_rank"] ad["global_rank_order"] = data[ad_image]["for_all_campaigns"]["global_rank_order"] with open(f"{os.environ.get('ULANMEDIAAPP')}/data/ads_for_one_campaign/{vol_id}_{date_range}_ads_for_one_campaign_dataset.json", "w") as file: json.dump(ads_for_one_campaign, file) return json.dumps(ads_for_one_campaign) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_ads_for_one_campaign_dataset()", "Failed - create_ads_for_one_campaign_dataset()") sys.exit()
def create_p_widgets_for_all_campaigns_dataset(date_range): try: # 1. get some prerequisite data campaigns = get_campaign_sets() with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_p_widgets/{date_range}_complete_p_widgets_dataset.json', 'r') as file: complete_p_widgets = json.load(file) ######################################################## # 2. set up the basic data structure you want to create p_widgets_for_all_campaigns = {"metadata":{}, "data":{}} ######################################################### # 3. Add the metadata. The metadata are the date ranges of the mgid and vol # request dates. All p_and_c_widgets_for_one_campaign files have the same # date ranges so I am just using the first campaign. vol_id_for_adding_metadata = campaigns[0]["vol_id"] with open(f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{campaigns[0]["vol_id"]}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json', 'r') as file: json_file = json.load(file) p_widgets_for_all_campaigns["metadata"]["mgid_start_date"] = json_file["metadata"]["mgid_start_date"] p_widgets_for_all_campaigns["metadata"]["mgid_end_date"] = json_file["metadata"]["mgid_end_date"] p_widgets_for_all_campaigns["metadata"]["vol_start_date"] = json_file["metadata"]["vol_start_date"] p_widgets_for_all_campaigns["metadata"]["vol_end_date"] = json_file["metadata"]["vol_end_date"] ######################################################### # 4. Add the data p_widgets_for_all_campaigns["data"] = complete_p_widgets ######################################################### # 5. remove "for_each_campaign" "good_campaigns_count" "bad_campaigns_count" and # "wait_campaigns_count" from each widget and add "good_campaigns_count" "bad_campaigns_count" # and "wait_campaigns_count" to p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"] for p_widget in p_widgets_for_all_campaigns["data"]: p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]["good_campaigns_count"] = p_widgets_for_all_campaigns["data"][p_widget]["good_campaigns_count"] p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]["bad_campaigns_count"] = p_widgets_for_all_campaigns["data"][p_widget]["bad_campaigns_count"] p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]["wait_campaigns_count"] = p_widgets_for_all_campaigns["data"][p_widget]["wait_campaigns_count"] p_widgets_for_all_campaigns["data"][p_widget] = p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"] ############################################################ # 6. Save p_widgets_for_all_campaigns to a json file and return it as a # json file with open(f"{os.environ.get('ULANMEDIAAPP')}/data/p_widgets_for_all_campaigns/{date_range}_p_widgets_for_all_campaigns_dataset.json", "w") as file: json.dump(p_widgets_for_all_campaigns, file) return json.dumps(p_widgets_for_all_campaigns) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_p_widgets_for_all_campaigns_dataset()", "Failed - create_p_widgets_for_all_campaigns_dataset()") sys.exit()
def create_gprs_for_each_p_offer_dataset(date_range): try: with open( f'{os.environ.get("ULANMEDIAAPP")}/data/offers_for_each_flow_rule/{date_range}_offers_for_each_flow_rule_dataset.json', 'r') as file: json_file = json.load(file) offers_for_each_flow_rule = json_file["data"] ##################################### gprs_for_each_p_offer = {} for flow_rule in offers_for_each_flow_rule: for offer in offers_for_each_flow_rule[flow_rule]: p_offer_name = offers_for_each_flow_rule[flow_rule][offer][ "p_offer_name"] gpr = offers_for_each_flow_rule[flow_rule][offer]["gpr"] p_offer_profit = round( offers_for_each_flow_rule[flow_rule][offer] ["p_offer_profit"], 2) p_offer_profit_rank = offers_for_each_flow_rule[flow_rule][ offer]["p_offer_profit_rank"] gpr_formula = offers_for_each_flow_rule[flow_rule][offer][ "gpr_formula"] roi_formula = offers_for_each_flow_rule[flow_rule][offer][ "roi_formula"] cvr_formula = offers_for_each_flow_rule[flow_rule][offer][ "cvr_formula"] if p_offer_name not in gprs_for_each_p_offer: gprs_for_each_p_offer[p_offer_name] = { "p_offer_name": p_offer_name, "gpr": gpr, "p_offer_profit": p_offer_profit, "p_offer_profit_rank": p_offer_profit_rank, "gpr_formula": gpr_formula, "roi_formula": roi_formula, "cvr_formula": cvr_formula } gprs_for_each_p_offer_list = [] for p_offer_name in gprs_for_each_p_offer: gprs_for_each_p_offer_list.append( gprs_for_each_p_offer[p_offer_name]) gprs_for_each_p_offer_list = pd.DataFrame(gprs_for_each_p_offer_list) gprs_for_each_p_offer_list = gprs_for_each_p_offer_list.sort_values( "p_offer_profit", ascending=True) return json.dumps(gprs_for_each_p_offer_list[[ "p_offer_profit_rank", "p_offer_profit", "p_offer_name", "gpr", "gpr_formula", "roi_formula", "cvr_formula" ]].to_dict("records")) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_gprs_for_each_p_offer_dataset()", "Failed - create_gprs_for_each_p_offer_dataset()") sys.exit()
def combine_mgid_vol_ads_data(mgid_token, vol_token, date_range, vol_start_date, vol_end_date, mgid_data, vol_data): try: # This function will combine the mgid ads data and the vol ads data. Both # data sets are dictionaries with keys = ad id and values = dictionaries of # data for that ad id. # create a look up dictionary so you can find vol id, and campaign name from mgid id campaigns_sets = get_campaign_sets() campaigns_lookup = {} for campaign in campaigns_sets: campaigns_lookup[campaign["mgid_id"]] = [ campaign["vol_id"], campaign["name"] ] # combining mgid and vol by ad_id combined_ads = { "metadata": { "vol_start_date": vol_start_date, "vol_end_date": vol_end_date }, "data": {} } for ad in mgid_data.values(): ad_id = ad["ad_id"] mgid_id = ad["mgid_id"] if mgid_id not in campaigns_lookup: continue vol_id = campaigns_lookup[mgid_id][0] name = campaigns_lookup[mgid_id][1] if ad_id in vol_data: vol_ad_data = vol_data[ad_id] ad["vol_id"] = vol_id ad["name"] = name ad["conversions"] = vol_ad_data["conversions"] ad["sales"] = vol_ad_data["sales"] ad["leads"] = vol_ad_data["leads"] ad["revenue"] = vol_ad_data["revenue"] else: ad["vol_id"] = vol_id ad["name"] = name ad["conversions"] = 0 ad["sales"] = 0 ad["leads"] = 0 ad["revenue"] = 0 combined_ads["data"][ad_id] = ad with open( f"{os.environ.get('ULANMEDIAAPP')}/data/ads/{date_range}_ads_dataset.json", "w") as file: json.dump(combined_ads, file) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - combine_mgid_vol_ads_data()", "Failed - combine_mgid_vol_ads_data()") sys.exit()
def create_months_for_one_ad_for_one_campaign_dataset(token, start_date, end_date, ad_image, campaign_id): try: start = datetime.strptime(start_date, "%Y-%m-%d").date() end = datetime.strptime(end_date, "%Y-%m-%d").date() months = {"metadata": {"vol_start_date": start_date, "vol_end_date": end_date}, "data": {}} url = f"https://api.voluum.com/report?from={start_date}T00:00:00Z&to={end_date}T00:00:00Z&tz=America%2FLos_Angeles&filter={ad_image}&conversionTimeMode=VISIT¤cy=USD&sort=month&direction=desc&columns=month&columns=customVariable5&columns=visits&columns=conversions&columns=revenue&columns=cost&columns=profit&columns=cv&columns=roi&columns=epv&groupBy=month&groupBy=custom-variable-5&offset=0&limit=1000&include=ACTIVE&filter1=campaign&filter1Value={campaign_id}" res = requests.get(url, headers = {"cwauth-token": token}).json() for row in res["rows"]: month = row["month"] clicks = row["visits"] cost = row["cost"] revenue = row["revenue"] profit = row["profit"] if month in months["data"]: months["data"][month]["clicks"] += clicks months["data"][month]["cost"] += cost months["data"][month]["revenue"] += revenue months["data"][month]["profit"] += profit else: months["data"][month] = {"clicks": clicks, "cost": cost, "revenue": revenue, "profit": profit, "leads": 0, "sales": 0, "month": month, "month_index": find_month_index_number(month) } with open(f'{os.environ.get("ULANMEDIAAPP")}/data/conversions_for_each_campaign/oneeighty_conversions_for_each_campaign_dataset.json', 'r') as file: json_file = json.load(file) conversions_for_each_campaign = json_file["data"] for conversion in conversions_for_each_campaign[campaign_id]: ad = conversion["customVariable5"] if ad != ad_image: continue month = int(conversion["visitTimestamp"].split('-')[1]) conversion_type = conversion["transactionId"] if conversion_type == "account": if month in months["data"]: months["data"][month]["leads"] += 1 elif conversion_type == "deposit": if month in months["data"]: months["data"][month]["sales"] += 1 with open(f"{os.environ.get('ULANMEDIAAPP')}/data/months_for_one_ad_for_one_campaign/{ad_image}_{campaign_id}_months_for_one_ad_for_one_campaign_dataset.json", "w") as file: json.dump(months, file) return json.dumps(months) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_months_for_one_ad_for_one_campaign_dataset()", "Failed - create_months_for_one_ad_for_one_campaign_dataset()") sys.exit()
def get_mgid_widget_clicks_and_costs_by_campaign(token, campaign_id, start_date, end_date): try: url = f"https://api.mgid.com/v1/goodhits/campaigns/{campaign_id}/quality-analysis?token={token}&campaignId={campaign_id}&dateInterval=interval&startDate={start_date}&endDate={end_date}" response = requests.get(url) if response.status_code == 401: mgid_token = get_and_return_new_mgid_token() return get_mgid_widget_clicks_and_costs_by_campaign( mgid_token, campaign_id, start_date, end_date) response.raise_for_status() response = response.json() # The logic below esstenially loops through each each widget and records # clicks into my dictionary of widgets. The only complication is if a widget # has sources, which are like child widgets. Sources are kept in a list. # For each source, if it is "0", that means it is really the parent widget, # and its data will be stored in voluum as such. If the source is not "0", # then it really is a child widget and its data is stored in voluum as # parent widget id + s + child widget id. # The reason why it matters how it is stored in voluum is because the # purpose of this function is to contribute to getting accurate widget # data. Mgid provides accurate data on clicks and cost and voluum provides # accurate data on the rest (conversions, revenue, etc). widgets_data = {} if response[campaign_id][start_date + "_" + end_date] == []: return widgets_data for id, data in response[campaign_id][start_date + "_" + end_date].items(): widget_id = id if data["sources"]: for source_id, source_data in data["sources"].items(): if source_id is not "0": widget_id = f"{id}s{source_id}" widgets_data[widget_id] = { "widget_id": widget_id, "clicks": source_data["clicks"], "cost": source_data["spent"], "coeff": source_data["qualityFactor"] } else: widgets_data[widget_id] = { "widget_id": widget_id, "clicks": data["clicks"], "cost": data["spent"], "coeff": data["qualityFactor"] } return widgets_data except requests.exceptions.RequestException as e: print("Failed - get_mgid_widget_clicks_and_costs_by_campaign") send_email( "*****@*****.**", "Failed - get_mgid_widget_clicks_and_costs_by_campaign() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_blacklist(): try: res = requests.get("https://ulanmedia.brianhaller.net/api/readblacklist") res.raise_for_status() return res.json() except requests.exceptions.RequestException as e: print("Failed to update campaign sets") print(e) send_email("*****@*****.**", "Failed - update_campaign_sets() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_vol_access_token(id, key): try: res = requests.post("https://api.voluum.com/auth/access/session", headers = {"Content-type": "application/json", "Accept": "application/json"}, data = json.dumps({"accessId": id, "accessKey": key})) res.raise_for_status() return res.json()["token"] except requests.exceptions.RequestException as e: print("Failed - get_vol_access_token()") send_email("*****@*****.**", "Failed - get_vol_access_token() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_all_campaign_revenues_by_traffic_source(token, traffic_source_id, start_date, end_date): url = f"https://api.voluum.com/report?from={start_date}T00%3A00%3A00Z&to={end_date}T00%3A00%3A00Z&tz={mgid_timezone}&sort=visits&direction=desc&columns=campaignName&columns=campaignId&columns=visits&columns=revenue&groupBy=campaign&offset=0&limit=100000&include=ACTIVE&conversionTimeMode=VISIT&filter1=traffic-source&filter1Value={traffic_source_id}" try: res = requests.get(url, headers={"cwauth-token": token}) res.raise_for_status() return res.json() except requests.exceptions.RequestException as e: print("Failed - get_all_campaign_revenues_by_traffic_source()") send_email( "*****@*****.**", "Failed - get_all_campaign_revenues_by_traffic_source at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_all_campaign_conversions_by_traffic_source(token, traffic_source_id, start_date, end_date): url=f"https://api.voluum.com/report/conversions?from={start_date}T00%3A00%3A00Z&to={end_date}T00:00:00Z&tz={mgid_timezone}&filter={traffic_source_id}&sort=campaignName&direction=asc&columns=transactionId&columns=revenue&columns=campaignName&columns=trafficSourceId&groupBy=conversion&offset=0&limit=100000&include=ACTIVE&conversionTimeMode=VISIT" try: campaigns = requests.get(url, headers = {"cwauth-token": token}).json() if campaigns["totalRows"] != len(campaigns["rows"]): # The get request has a limit of 100,000 so if there are more than # 100,000 conversions returned, this exception will be raised. raise Exception("voluum didn't return all the conversion data.") # create a dictionary to hold the conversion and revenue data for each # campaign. Fill the empty campaigns_data with key = vol_id and value = # dict with name, revenue, leads, and sales keys. campaigns_data = {} for campaign in campaigns["rows"]: vol_id = campaign["campaignId"] campaign_name = re.sub(r"^.* - ", "",campaign["campaignName"], count=1) if vol_id not in campaigns_data: campaigns_data[vol_id] = {"name": campaign_name, "revenue": 0, "leads": 0, "sales": 0} for campaign in campaigns["rows"]: # check that visitTimestamp is within the date range of choice. # This is necessary because the request returns results based on # the sale date, not the original click date. We only want results # where the original click date is within the chosen date range. start_date_in_date_format = datetime.strptime(start_date, "%Y-%m-%d") click_date_in_date_format = datetime.strptime(campaign["visitTimestamp"], "%Y-%m-%d %I:%M:%S %p") if start_date_in_date_format > click_date_in_date_format: continue revenue = campaign["revenue"] vol_id = campaign["campaignId"] campaign_name = re.sub(r"^.* - ", "",campaign["campaignName"], count=1) lead = 0 sale = 0 if campaign["transactionId"] == "account": lead = 1 elif campaign["transactionId"] == "deposit": sale = 1 campaigns_data[vol_id]["leads"] += lead campaigns_data[vol_id]["sales"] += sale campaigns_data[vol_id]["revenue"] += revenue return campaigns_data except requests.exceptions.RequestException as e: print("Failed - get_all_campaign_conversions_by_traffic_source()") send_email("*****@*****.**", "Failed - get_all_campaign_conversions_by_traffic_source at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def exclude_all_campaigns_for_blacklisted_p_widgets(date_range): with open( f'{os.environ.get("ULANMEDIAAPP")}/data/complete_p_widgets/{date_range}_complete_p_widgets_dataset.json', 'r') as file: complete_p_widgets = json.load(file) emails_sent = 0 for p_widget in complete_p_widgets.values(): if p_widget["for_all_campaigns"]["global_status"] == "p_blacklist": for campaign in p_widget["for_each_campaign"]: if campaign["status"] == "included": subject = 'ALERT - found an "included" campaign in a blacklisted widget' message = f'found an included campaign on a blacklisted p widget:\np widget id {p_widget["for_all_campaigns"]["widget_id"]}\ncampaign {campaign["mgid_id"]}\nhttps://ulanmedia.brianhaller.net/campaignsforonepwidget/{p_widget["for_all_campaigns"]["widget_id"]}' print(message) emails_sent += 1 send_email("*****@*****.**", subject, message) send_email("*****@*****.**", subject, message) if emails_sent == 0: # send email saying no "included" campaigns on blacklisted widgets were found. message = 'ok - campaigns for blacklisted widgets are all excluded' print(message) send_email("*****@*****.**", message, message) send_email("*****@*****.**", message, message)
def create_offers_for_one_flow_rule_dataset(date_range, flow_rule_argument): try: with open( f'{os.environ.get("ULANMEDIAAPP")}/data/offers_for_each_flow_rule/{date_range}_offers_for_each_flow_rule_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] offers_for_each_flow_rule = json_file["data"] offers_for_one_flow_rule = { "metadata": metadata, "data": offers_for_each_flow_rule[flow_rule_argument] } ####################################################### # Add classification to each offer for offer in offers_for_one_flow_rule["data"]: offers_for_one_flow_rule["data"][offer][ "classification"] = classify_offer_for_all_campaigns( offers_for_one_flow_rule["data"][offer]) ####################################################### # Add has_mismatch_vol_weight_and_rec_weight to each offer for offer in offers_for_one_flow_rule["data"]: vol_weight = offers_for_one_flow_rule["data"][offer]["vol_weight"] rec_weight = offers_for_one_flow_rule["data"][offer]["rec_weight"] if (vol_weight != rec_weight) & (vol_weight != "NA"): offers_for_one_flow_rule["data"][offer][ "has_mismatch_vol_weight_and_rec_weight"] = True else: offers_for_one_flow_rule["data"][offer][ "has_mismatch_vol_weight_and_rec_weight"] = False ############################################### # Save file and return with open( f"{os.environ.get('ULANMEDIAAPP')}/data/offers_for_one_flow_rule/{flow_rule_argument}_{date_range}_offers_for_one_flow_rule_dataset.json", "w") as file: json.dump(offers_for_one_flow_rule, file) return json.dumps(offers_for_one_flow_rule) except: print("Failed - email sent") send_email("*****@*****.**", "Failed - create_offers_for_one_flow_rule_dataset()", "Failed - create_offers_for_one_flow_rule_dataset()") sys.exit()
def get_vol_ads_data(date_range): try: with open(f'{os.environ.get("ULANMEDIAAPP")}/data/conversions_for_each_campaign/{date_range}_conversions_for_each_campaign_dataset.json', 'r') as file: json_file = json.load(file) metadata = json_file["metadata"] conversions_for_each_campaign = json_file["data"] # campaigns will be a list of dictionaries, where each dictionary is a # campaign with keys and values like {vol_id: "123", mgid_id: 123, ...} campaigns = get_campaign_sets() # The ads_data dictionary will have this structure: each key is an ad id # (customvariable3) and each value is a dictionary with stats for that ad ads_data = {} for campaign in campaigns: vol_id = campaign["vol_id"] name = campaign["name"] if vol_id not in conversions_for_each_campaign: continue for conversion in conversions_for_each_campaign[vol_id]: ad_id = conversion["customVariable3"] if ad_id not in ads_data: ads_data[ad_id] = {} ads_data[ad_id]["ad_id"] = ad_id ads_data[ad_id]["vol_id"] = vol_id ads_data[ad_id]["name"] = name ads_data[ad_id]["revenue"] = conversion["revenue"] ads_data[ad_id]["conversions"] = 1 if conversion["transactionId"] == "account": ads_data[ad_id]["leads"] = 1 ads_data[ad_id]["sales"] = 0 elif conversion["transactionId"] == "deposit": ads_data[ad_id]["sales"] = 1 ads_data[ad_id]["leads"] = 0 else: ads_data[ad_id]["revenue"] += conversion["revenue"] ads_data[ad_id]["conversions"] += 1 if conversion["transactionId"] == "account": ads_data[ad_id]["leads"] += 1 elif conversion["transactionId"] == "deposit": ads_data[ad_id]["sales"] += 1 return ads_data except: print("Failed - email sent") send_email("*****@*****.**", "Failed - get_vol_ads_data()", "Failed - get_vol_ads_data()") sys.exit()
def check_all_mgid_ads(token): request_number = 0 ads_data = {} while len(ads_data) == request_number * 700: url = f"https://api.mgid.com/v1/goodhits/clients/{mgid_client_id}/teasers?token={token}&limit=700&start={request_number * 700}" request_number = request_number + 1 res = requests.get(url) if res.status_code == 401: mgid_token = get_and_return_new_mgid_token() return check_all_mgid_ads(mgid_token) res.raise_for_status() res = res.json() if len(res) == 0: break for ad_id in res: ads_data[ad_id] = res[ad_id] ad_status_counts = {"active": 0, "paused": 0, "pending": 0, "rejected": 0} rejected_ads = {} for ad_id in ads_data: if (ads_data[ad_id]["status"]["code"] == 'goodPerformance'): ad_status_counts["active"] += 1 elif (ads_data[ad_id]["status"]["code"] == 'new'): ad_status_counts["active"] += 1 elif (ads_data[ad_id]["status"]["code"] == 'campaignBlocked'): ad_status_counts["paused"] += 1 elif (ads_data[ad_id]["status"]["code"] == 'blocked'): ad_status_counts["paused"] += 1 elif (ads_data[ad_id]["status"]["code"] == 'onModeration'): ad_status_counts["pending"] += 1 else: ad_status_counts["rejected"] += 1 rejected_ads[ad_id] = { "ad_id": ad_id, "campaign_id": ads_data[ad_id]["campaignId"] } subject = f"ok - {len(ads_data)} ads: {ad_status_counts['active']} active, {ad_status_counts['paused']} paused, {ad_status_counts['pending']} pending, {ad_status_counts['rejected']} rejected" body = f"{ad_status_counts['active']} of {len(ads_data)} ads in all campaigns are active.\n{ad_status_counts['paused']} of {len(ads_data)} ads in all campaigns are paused.\n{ad_status_counts['pending']} of {len(ads_data)} ads in all campaigns are pending.\n{ad_status_counts['rejected']} of {len(ads_data)} ads in all campaigns are rejected." if ad_status_counts["rejected"] > 0: subject = f"ALERT - {len(ads_data)} ads: {ad_status_counts['active']} active, {ad_status_counts['paused']} paused, {ad_status_counts['pending']} pending, {ad_status_counts['rejected']} rejected" for ad_id in rejected_ads: body += f'\n\nad {rejected_ads[ad_id]["ad_id"]} rejected in campaign {rejected_ads[ad_id]["campaign_id"]}\nhttps://dashboard.mgid.com/advertisers/teasers-goods/campaign_id/{rejected_ads[ad_id]["campaign_id"]}' print(subject) print(body) send_email("*****@*****.**", subject, body) send_email("*****@*****.**", subject, body)
def get_mgid_campaign_costs(token, client_id, start, end): try: res = requests.get( f"https://api.mgid.com/v1/goodhits/clients/{client_id}/campaigns-stat?token={token}&dateInterval=interval&startDate={start}&endDate={end}" ) if res.status_code == 401: mgid_token = get_and_return_new_mgid_token() return get_mgid_campaign_costs(mgid_token, client_id, start, end) res.raise_for_status() return res.json() except requests.exceptions.RequestException as e: print("Failed - get_mgid_campaign_costs") send_email( "*****@*****.**", "Failed - get_mgid_campaign_costs() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_mgid_daily_stats_data(token, start_date, end_date): mgid_url = f"https://api.mgid.com/v1/goodhits/clients/{mgid_client_id}/campaigns-stat?token={token}&dateInterval=interval&startDate={start_date}&endDate={end_date}" try: res = requests.get(mgid_url) if res.status_code == 401: mgid_token = get_and_return_new_mgid_token() return get_mgid_daily_stats_data(mgid_token, start_date, end_date) res.raise_for_status() return res.json()["campaigns-stat"] except requests.exceptions.RequestException as e: send_email( "*****@*****.**", "Failed - get_mgid_daily_stats_data at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def update_widget_domains_file(): try: res = requests.get("http://ulanmedia.com/mgid/widgetdomains.txt") res.raise_for_status() res = res.text.split("\n") widget_domains = [] for widget_domain in res: widget_domains.append(widget_domain.replace("\r", "")) widget_domain_lookup = {} for widget_domain in widget_domains: if len(widget_domain) > 0: split_widget_domain = widget_domain.split(",") if len(split_widget_domain) != 4: continue traffic_source = split_widget_domain[0] widget_id = split_widget_domain[1] domain = split_widget_domain[2] widget_domain_source = split_widget_domain[3] if widget_id in widget_domain_lookup: widget_domain_lookup[widget_id]['domains'].append(domain) else: widget_domain_lookup[widget_id] = { 'widget_id': widget_id, 'domains': [domain], 'traffic_source': traffic_source, 'widget_domain_source': widget_domain_source } with open( f"{os.environ.get('ULANMEDIAAPP')}/curated_lists/widget_domains/widget_domains.json", "w") as file: json.dump(widget_domain_lookup, file) except requests.exceptions.RequestException as e: print("Failed to update widget domains file") print(e) send_email( "*****@*****.**", "Failed - update_widget_domains_file() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def create_active_flow_rules_list(token): try: flow_id = "da8f9291-462d-43e7-98a4-24d62f608297" url = f"https://api.voluum.com/flow/{flow_id}" response = requests.get(url, headers = {"cwauth-token": token}) response.raise_for_status() response = response.json() active_flow_rules = [] for offer in response["defaultPaths"]: active_flow_rules.append(offer["name"]) for offer in response["conditionalPathsGroups"]: active_flow_rules.append(offer["name"]) with open(f"../../data/active_flow_rules/active_flow_rules.json", "w") as file: json.dump(active_flow_rules, file) except requests.exceptions.RequestException as e: print("Failed - create_active_flow_rules_list") send_email("*****@*****.**", "Failed - create_active_flow_rules_list() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_vol_daily_stats_data(token, start_date, end_date, timezone): vol_url = f"https://api.voluum.com/report?from={start_date}T00%3A00%3A00Z&to={end_date}T00:00:00Z&tz={timezone}&sort=campaignName&direction=desc&columns=campaignName&columns=day&columns=campaignId&columns=visits&columns=conversions&columns=revenue&columns=cost&columns=cpv&groupBy=campaign&groupBy=day&offset=0&limit=1000000&include=ACTIVE&conversionTimeMode=VISIT&filter1=traffic-source&filter1Value={mgidVolTrafficSourceId}" try: res = requests.get(vol_url, headers={"cwauth-token": token}) res.raise_for_status() vol_response = res.json() if vol_response["totalRows"] != len(vol_response["rows"]): send_email( "*****@*****.**", "Failed - get_vol_daily_stats_data at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), "all data not returned from voluum") raise Exception("voluum didn't return all the conversion data") return vol_response["rows"] except requests.exceptions.RequestException as e: print("exception handled") send_email( "*****@*****.**", "Failed - get_vol_daily_stats_data at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def get_mgid_included_widgets_by_campaign(token, campaign_id, start_date, end_date): try: url = f"https://api.mgid.com/v1/goodhits/campaigns/{campaign_id}/quality-analysis?token={token}&campaignId={campaign_id}&dateInterval=interval&startDate={start_date}&endDate={end_date}" response = requests.get(url) if response.status_code == 401: mgid_token = get_and_return_new_mgid_token() return get_mgid_included_widgets_by_campaign( mgid_token, campaign_id, start_date, end_date) response.raise_for_status() response = response.json() widgets = [] if response[campaign_id][start_date + "_" + end_date] == []: return widgets for id, data in response[campaign_id][start_date + "_" + end_date].items(): widget_id = id widgets.append(widget_id) if data["sources"]: for source_id, source_data in data["sources"].items(): if source_id is not "0": widget_id = f"{id}s{source_id}" widgets.append(widget_id) excluded_widgets = get_mgid_excluded_widgets_by_campaign( token, mgid_client_id, campaign_id) included_widgets = [] for widget in widgets: if widget not in excluded_widgets: included_widgets.append(widget) return included_widgets except: print("Failed - email sent") send_email("*****@*****.**", "Failed - get_mgid_included_widgets_by_campaign()", "Failed - get_mgid_included_widgets_by_campaign()") sys.exit()
def get_mgid_excluded_widgets_by_campaign(mgid_token, mgid_client_id, mgid_campaign_id): url = f"https://api.mgid.com/v1/goodhits/clients/{mgid_client_id}/campaigns/{mgid_campaign_id}?token={mgid_token}" try: response = requests.get(url) if response.status_code == 401: new_mgid_token = get_and_return_new_mgid_token() return get_mgid_excluded_widgets_by_campaign( new_mgid_token, mgid_client_id, mgid_campaign_id) response.raise_for_status() response = response.json() # the response is a dictionary of lists. The keys are widget ids. # The values are sources. If a widget id has a source, it must be # this concatenation is necessary {widget_id}s{source_id}. # One strange thing about the response is that the source id appears # to be in a list but the [] are actually just part of the source id # string. excluded_widgets = [] if response["widgetsFilterUid"]["widgets"] == []: return excluded_widgets for key, value in response["widgetsFilterUid"]["widgets"].items(): if (value == "[]") | (value == None): excluded_widgets.append(key) else: value = value.replace("[", "") value = value.replace("]", "") value = value.replace(",", "") c_widgets = value.split(" ") for c_widget in c_widgets: excluded_widgets.append(f"{key}s{c_widget}") # excluded_widgets is a list of excluded widget ids return excluded_widgets except requests.exceptions.RequestException as e: print("Failed - get_mgid_excluded_widgets_by_campaign") send_email( "*****@*****.**", "Failed - get_mgid_excluded_widgets_by_campaign() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def update_good_widgets_file(): try: res = requests.get("http://ulanmedia.com/mgid/goodpwidgets.txt") res.raise_for_status() widgets = res.text.split("\n") good_widgets = [] for widget in widgets: if widget != "": good_widgets.append(widget.replace("\r", "")) with open( f"{os.environ.get('ULANMEDIAAPP')}/curated_lists/good_widgets/good_widgets.json", "w") as file: json.dump(good_widgets, file) except requests.exceptions.RequestException as e: print("Failed to update good widgets file") print(e) send_email( "*****@*****.**", "Failed - update_good widgets_file() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()
def create_conversions_for_each_campaign_dataset(token, start_date, end_date, date_range): try: url = f"https://api.voluum.com/report/conversions?from={start_date}T00:00:00Z&to={end_date}T00:00:00Z&tz=America%2FLos_Angeles&conversionTimeMode=VISIT&sort=visitTimestamp&direction=desc&columns=visitTimestamp&columns=transactionId&columns=campaignId&columns=offerId&columns=countryName&columns=trafficSourceId&columns=deviceName&columns=os&columns=browser&columns=isp&columns=customVariable1&columns=customVariable3&groupBy=conversion&offset=0&limit=1000000&include=ACTIVE&filter=37bbd390-ed90-4978-9066-09affa682bcc" res = requests.get(url, headers = {"cwauth-token": token}) res.raise_for_status() res = res.json() conversions_for_each_campaign = {"metadata": {"vol_start_date": start_date, "vol_end_date": end_date}, "data": {}} for conversion in res["rows"]: campaign_id = conversion["campaignId"] if campaign_id not in conversions_for_each_campaign["data"]: conversions_for_each_campaign["data"][campaign_id] = [conversion] else: conversions_for_each_campaign["data"][campaign_id].append(conversion) with open(f"{os.environ.get('ULANMEDIAAPP')}/data/conversions_for_each_campaign/{date_range}_conversions_for_each_campaign_dataset.json", "w") as file: json.dump(conversions_for_each_campaign, file) except requests.exceptions.RequestException as e: print("Failed - create_conversions_for_each_campaign_dataset()") print(e) send_email("*****@*****.**", "Failed - create_conversions_for_each_campaign_dataset() at " + str(datetime.now().strftime("%Y-%m-%d %H:%M")), e) sys.exit()