def create_campaigns_for_one_country_dataset(date_range, country_name):
try:
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_countries/{date_range}_complete_countries_dataset.json', 'r') as file:
json_file = json.load(file)
metadata = json_file["metadata"]
data = json_file["data"]
campaigns_for_one_country = {"metadata": metadata, "data": data[country_name]["for_each_campaign"]}
campaigns_sets = get_campaign_sets()
campaigns_lookup = {}
for campaign in campaigns_sets:
campaigns_lookup[campaign["vol_id"]] = campaign["name"]
for campaign_id in campaigns_for_one_country["data"]:
campaigns_for_one_country["data"][campaign_id]["campaign_name"] = campaigns_lookup[campaign_id]
with open(f"{os.environ.get('ULANMEDIAAPP')}/data/campaigns_for_one_country/{date_range}_{country_name}_campaigns_for_one_country_dataset.json", "w") as file:
json.dump(campaigns_for_one_country, file)
return json.dumps(campaigns_for_one_country)
except:
print("Failed - email sent")
send_email("*****@*****.**", "Failed - create_campaigns_for_one_country_dataset()", "Failed - create_campaigns_for_one_country_dataset()")
sys.exit()
def combine_mgid_vol_ads_data(mgid_token, vol_token, date_range,
vol_start_date, vol_end_date, mgid_data,
vol_data):
try:
# This function will combine the mgid ads data and the vol ads data. Both
# data sets are dictionaries with keys = ad id and values = dictionaries of
# data for that ad id.
# create a look up dictionary so you can find vol id, and campaign name from mgid id
campaigns_sets = get_campaign_sets()
campaigns_lookup = {}
for campaign in campaigns_sets:
campaigns_lookup[campaign["mgid_id"]] = [
campaign["vol_id"], campaign["name"]
]
# combining mgid and vol by ad_id
combined_ads = {
"metadata": {
"vol_start_date": vol_start_date,
"vol_end_date": vol_end_date
},
"data": {}
}
for ad in mgid_data.values():
ad_id = ad["ad_id"]
mgid_id = ad["mgid_id"]
if mgid_id not in campaigns_lookup:
continue
vol_id = campaigns_lookup[mgid_id][0]
name = campaigns_lookup[mgid_id][1]
if ad_id in vol_data:
vol_ad_data = vol_data[ad_id]
ad["vol_id"] = vol_id
ad["name"] = name
ad["conversions"] = vol_ad_data["conversions"]
ad["sales"] = vol_ad_data["sales"]
ad["leads"] = vol_ad_data["leads"]
ad["revenue"] = vol_ad_data["revenue"]
else:
ad["vol_id"] = vol_id
ad["name"] = name
ad["conversions"] = 0
ad["sales"] = 0
ad["leads"] = 0
ad["revenue"] = 0
combined_ads["data"][ad_id] = ad
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/ads/{date_range}_ads_dataset.json",
"w") as file:
json.dump(combined_ads, file)
except:
print("Failed - email sent")
send_email("*****@*****.**",
"Failed - combine_mgid_vol_ads_data()",
"Failed - combine_mgid_vol_ads_data()")
sys.exit()
def create_p_widgets_for_all_campaigns_dataset(date_range):
try:
# 1. get some prerequisite data
campaigns = get_campaign_sets()
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_p_widgets/{date_range}_complete_p_widgets_dataset.json', 'r') as file:
complete_p_widgets = json.load(file)
########################################################
# 2. set up the basic data structure you want to create
p_widgets_for_all_campaigns = {"metadata":{}, "data":{}}
#########################################################
# 3. Add the metadata. The metadata are the date ranges of the mgid and vol
# request dates. All p_and_c_widgets_for_one_campaign files have the same
# date ranges so I am just using the first campaign.
vol_id_for_adding_metadata = campaigns[0]["vol_id"]
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{campaigns[0]["vol_id"]}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json', 'r') as file:
json_file = json.load(file)
p_widgets_for_all_campaigns["metadata"]["mgid_start_date"] = json_file["metadata"]["mgid_start_date"]
p_widgets_for_all_campaigns["metadata"]["mgid_end_date"] = json_file["metadata"]["mgid_end_date"]
p_widgets_for_all_campaigns["metadata"]["vol_start_date"] = json_file["metadata"]["vol_start_date"]
p_widgets_for_all_campaigns["metadata"]["vol_end_date"] = json_file["metadata"]["vol_end_date"]
#########################################################
# 4. Add the data
p_widgets_for_all_campaigns["data"] = complete_p_widgets
#########################################################
# 5. remove "for_each_campaign" "good_campaigns_count" "bad_campaigns_count" and
# "wait_campaigns_count" from each widget and add "good_campaigns_count" "bad_campaigns_count"
# and "wait_campaigns_count" to p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]
for p_widget in p_widgets_for_all_campaigns["data"]:
p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]["good_campaigns_count"] = p_widgets_for_all_campaigns["data"][p_widget]["good_campaigns_count"]
p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]["bad_campaigns_count"] = p_widgets_for_all_campaigns["data"][p_widget]["bad_campaigns_count"]
p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]["wait_campaigns_count"] = p_widgets_for_all_campaigns["data"][p_widget]["wait_campaigns_count"]
p_widgets_for_all_campaigns["data"][p_widget] = p_widgets_for_all_campaigns["data"][p_widget]["for_all_campaigns"]
############################################################
# 6. Save p_widgets_for_all_campaigns to a json file and return it as a
# json file
with open(f"{os.environ.get('ULANMEDIAAPP')}/data/p_widgets_for_all_campaigns/{date_range}_p_widgets_for_all_campaigns_dataset.json", "w") as file:
json.dump(p_widgets_for_all_campaigns, file)
return json.dumps(p_widgets_for_all_campaigns)
except:
print("Failed - email sent")
send_email("*****@*****.**", "Failed - create_p_widgets_for_all_campaigns_dataset()", "Failed - create_p_widgets_for_all_campaigns_dataset()")
sys.exit()
def calculate_max_numbers_for_all_campaigns():
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/campaigns_for_all_campaigns/oneeighty_campaigns_for_all_campaigns_dataset.json',
'r') as file:
json_file = json.load(file)
# create a dictionary of campaign sets
# keys are vol_id, values are all the data for each campaign in campaign
# sets
campaigns = get_campaign_sets()
campaigns_dict = {}
for campaign in campaigns:
campaigns_dict[campaign["vol_id"]] = campaign
for campaign in json_file["data"]:
name = campaign["name"]
vol_id = campaign["vol_id"]
revenue = campaign["revenue"]
clicks = campaign["clicks"]
leads = campaign["leads"]
sales = campaign["sales"]
print("###################")
print(f"campaign name: {name}")
if clicks == 0:
print(f"recommended maxBid (EPC): 0")
else:
print(f"recommended maxBid (EPC): {revenue/clicks}")
if leads == 0:
print(f"recommended maxLeadCPA (EPL): 0")
print(
f"current maxLeadCPA: {campaigns_dict[vol_id]['max_lead_cpa']}"
)
else:
print(f"recommended maxLeadCPA (EPL): {revenue/leads}")
print(
f"current maxLeadCPA: {campaigns_dict[vol_id]['max_lead_cpa']}"
)
if sales == 0:
print(f"recommended maxSaleCPA = EPS: 0")
print(
f"current maxSaleCPA: {campaigns_dict[vol_id]['max_sale_cpa']}"
)
else:
print(f"recommended maxSaleCPA = EPS: {revenue/sales}")
print(
f"current maxSaleCPA: {campaigns_dict[vol_id]['max_sale_cpa']}"
)
print("###################")
def get_vol_ads_data(date_range):
try:
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/conversions_for_each_campaign/{date_range}_conversions_for_each_campaign_dataset.json', 'r') as file:
json_file = json.load(file)
metadata = json_file["metadata"]
conversions_for_each_campaign = json_file["data"]
# campaigns will be a list of dictionaries, where each dictionary is a
# campaign with keys and values like {vol_id: "123", mgid_id: 123, ...}
campaigns = get_campaign_sets()
# The ads_data dictionary will have this structure: each key is an ad id
# (customvariable3) and each value is a dictionary with stats for that ad
ads_data = {}
for campaign in campaigns:
vol_id = campaign["vol_id"]
name = campaign["name"]
if vol_id not in conversions_for_each_campaign:
continue
for conversion in conversions_for_each_campaign[vol_id]:
ad_id = conversion["customVariable3"]
if ad_id not in ads_data:
ads_data[ad_id] = {}
ads_data[ad_id]["ad_id"] = ad_id
ads_data[ad_id]["vol_id"] = vol_id
ads_data[ad_id]["name"] = name
ads_data[ad_id]["revenue"] = conversion["revenue"]
ads_data[ad_id]["conversions"] = 1
if conversion["transactionId"] == "account":
ads_data[ad_id]["leads"] = 1
ads_data[ad_id]["sales"] = 0
elif conversion["transactionId"] == "deposit":
ads_data[ad_id]["sales"] = 1
ads_data[ad_id]["leads"] = 0
else:
ads_data[ad_id]["revenue"] += conversion["revenue"]
ads_data[ad_id]["conversions"] += 1
if conversion["transactionId"] == "account":
ads_data[ad_id]["leads"] += 1
elif conversion["transactionId"] == "deposit":
ads_data[ad_id]["sales"] += 1
return ads_data
except:
print("Failed - email sent")
send_email("*****@*****.**", "Failed - get_vol_ads_data()", "Failed - get_vol_ads_data()")
sys.exit()
def create_offers_for_each_campaign_dataset(token, date_range, vol_start_date,
vol_end_date):
# Overview of how this function works
# 1. get the flow rules and offers from https://api.voluum.com/flow/{vol_flow_id}
# 2. get the offers from https://api.voluum.com/report....
# 3. get the flow rule names from the parenthesis from each offer name.
# 4. get the voluum flow index numbers to display on the dashboard in the same order as in voluum.
# 5. get the current flow offer weights from the voluum api flow url.
# 6. get the offer stats from the voluum api offers report url.
try:
url = f"https://api.voluum.com/report?from={vol_start_date}T00%3A00%3A00Z&to={vol_end_date}T00%3A00%3A00Z&tz=America%2FLos_Angeles&conversionTimeMode=VISIT&sort=offerName&direction=desc&columns=offerName&columns=campaignName&columns=visits&columns=conversions&columns=revenue&columns=cost&columns=profit&columns=cpv&columns=cv&columns=roi&columns=epv&columns=campaignId&columns=cpa&groupBy=offer&groupBy=campaign&offset=0&limit=100000&include=ACTIVE&filter1=traffic-source&filter1Value=37bbd390-ed90-4978-9066-09affa682bcc"
res = requests.get(url, headers={"cwauth-token": token})
res.raise_for_status()
if res.json()["totalRows"] != len(res.json()["rows"]):
# you need to throw an error here
print("problem")
sys.exit()
############################3
# create an offer weight lookup dictionary and a flow rule index
# lookup (for ordering the flow rules the same on dashboard as in
# voluum)
vol_weight_url = f"https://api.voluum.com/flow/{vol_flow_id}"
vol_weight_res = requests.get(vol_weight_url,
headers={"cwauth-token": token})
vol_weight_res.raise_for_status()
vol_weight_res = vol_weight_res.json()
vol_weight_lookup = {}
for offer in vol_weight_res["defaultPaths"][0]["offers"]:
offer_id = offer["offer"]["id"]
if offer_id not in vol_weight_lookup:
vol_weight_lookup[offer_id] = offer["weight"]
else:
print("there shouldn't be any repeats")
for path_group in vol_weight_res["conditionalPathsGroups"]:
for offer in path_group["paths"][0]["offers"]:
offer_id = offer["offer"]["id"]
if offer_id not in vol_weight_lookup:
vol_weight_lookup[offer_id] = offer["weight"]
else:
print("there shouldn't be any repeats")
vol_flow_rule_index_lookup = {}
index = 1
for flow_rule in vol_weight_res["conditionalPathsGroups"]:
vol_flow_rule_index_lookup[flow_rule["name"]] = index
index += 1
vol_flow_rule_index_lookup["default"] = index
########################
offers = {
"metadata": {
"vol_start_date": vol_start_date,
"vol_end_date": vol_end_date
},
"data": {}
}
for row in res.json()["rows"]:
campaign_id = row["campaignId"]
if campaign_id not in offers["data"]:
offers["data"][campaign_id] = {}
campaigns = get_campaign_sets()
campaign_ids = []
for campaign in campaigns:
campaign_ids.append(campaign["vol_id"])
for row in res.json()["rows"]:
campaign_id = row["campaignId"]
offer_id = row["offerId"]
if (campaign_id not in campaign_ids) | (
row["offerName"].startswith("Global - 404")):
continue
else:
# 11/06/19 This conditional was added because mike updated his naming convention for offer names and flow rule names to include the landin page within square brackets. The old inactive offers don't have square brackets, which breaks the regex, but their data is unneeded anyway, so it is okay to skip them.
if ']' not in row["offerName"]:
continue
pattern = re.compile(r'(^\w* - \w* - {1})(.[^(]*) (.*)')
res = pattern.findall(row["offerName"])
offer_string_parts = list(res[0])
c_offer_name = offer_string_parts[1]
flow_rule = offer_string_parts[2]
##############################
# find parent_offer_name
country_list = ["spain", "latam", "brazil", "portugal"]
tier_list = ["tier1", "tier3"]
offer_words = c_offer_name.split(" ")
# remove the country code at the end (eg. DE)
offer_words.pop()
# remove the country name if there is one (eg. spain)
if offer_words[len(offer_words) - 1] in country_list:
offer_words.pop()
# remove the tier if there is one (eg. tier1)
if offer_words[len(offer_words) - 1] in tier_list:
offer_words.pop()
p_offer_name = " ".join(offer_words)
if offer_id not in vol_weight_lookup:
vol_weight = "NA"
else:
vol_weight = vol_weight_lookup[offer_id]
if offer_id not in offers["data"][campaign_id]:
if offer_id in vol_weight_lookup and vol_weight_lookup[
offer_id] != 0:
if flow_rule not in vol_flow_rule_index_lookup:
# 5/8/19 I would like to know which flow rules are being
# skipped but there is a very large number - why?
# send_email("*****@*****.**", "Failed - create_offers_for_each_campaign_dataset() at "
# + str(datetime.now().strftime("%Y-%m-%d%H:%M")),
# f'the flow rule name "{flow_rule}" is not in the voluum flow rules. There is likely a typo.')
continue
offers["data"][campaign_id][offer_id] = {
"clicks": row["visits"],
"cost": row["cost"],
"offer_id": offer_id,
"profit": row["profit"],
"revenue": row["profit"] + row["cost"],
"conversions": row["conversions"],
# leads and sales are added in
# the next step
"leads": 0,
"sales": 0,
"offer_id": offer_id,
"campaign_id": campaign_id,
"vol_offer_name": row["offerName"],
"offer_name": f"{c_offer_name} {flow_rule}",
"p_offer_name": p_offer_name,
"c_offer_name": c_offer_name,
"flow_rule": flow_rule,
"flow_rule_index":
vol_flow_rule_index_lookup[flow_rule],
"vol_weight": vol_weight
}
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/conversions_for_each_campaign/{date_range}_conversions_for_each_campaign_dataset.json',
'r') as file:
json_file = json.load(file)
metadata = json_file["metadata"]
conversions_for_each_campaign = json_file["data"]
for campaign_id in conversions_for_each_campaign:
if (campaign_id not in campaign_ids):
continue
for conversion in conversions_for_each_campaign[campaign_id]:
offer_id = conversion["offerId"]
if offer_id in offers["data"][campaign_id]:
if conversion["transactionId"] == "account":
offers["data"][campaign_id][offer_id]["leads"] += 1
elif conversion["transactionId"] == "deposit":
offers["data"][campaign_id][offer_id]["sales"] += 1
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/offers_for_each_campaign/{date_range}_offers_for_each_campaign_dataset.json",
"w") as file:
json.dump(offers, file)
except requests.exceptions.RequestException as e:
print("Failed - create_offers_for_each_campaign_dataset()")
send_email(
"*****@*****.**",
"Failed - create_offers_for_each_campaign_dataset() at " +
str(datetime.now().strftime("%Y-%m-%d %H:%M")), e)
sys.exit()
from functions.misc.get_campaign_sets import get_campaign_sets campaigns = get_campaign_sets() print(campaigns)
def create_languages_dataset(token, start_date, end_date, date_range):
try:
url = f"https://api.voluum.com/report?from={start_date}T00:00:00Z&to={end_date}T00:00:00Z&tz=America%2FLos_Angeles&conversionTimeMode=VISIT&sort=campaignName&direction=asc&columns=campaignName&columns=languageName&columns=campaignId&columns=visits&columns=conversions&columns=revenue&columns=cost&columns=profit&columns=cpv&columns=cv&columns=roi&columns=epv&columns=cpa&groupBy=campaign&groupBy=language&offset=0&limit=100000&include=ACTIVE&filter1=traffic-source&filter1Value=37bbd390-ed90-4978-9066-09affa682bcc"
res = requests.get(url, headers={"cwauth-token": token})
res.raise_for_status()
res = res.json()
languages = {
"metadata": {
"vol_start_date": start_date,
"vol_end_date": end_date
},
"data": {}
}
campaigns = get_campaign_sets()
vol_ids = []
for campaign in campaigns:
vol_ids.append(campaign["vol_id"])
for row in res["rows"]:
language_name = row["languageName"]
if language_name == "Spanish/Castilian":
language_name = language_name.replace("/", ", ")
campaign_id = row["campaignId"]
if campaign_id not in vol_ids:
continue
if language_name not in languages["data"]:
languages["data"][language_name] = {
campaign_id: {
"campaign_id": campaign_id,
"language_name": language_name,
"clicks": row["visits"],
"conversions": row["conversions"],
"cost": row["cost"],
"profit": row["profit"],
"revenue": row["revenue"]
}
}
else:
languages["data"][language_name][campaign_id] = {
"campaign_id": campaign_id,
"language_name": language_name,
"clicks": row["visits"],
"conversions": row["conversions"],
"cost": row["cost"],
"profit": row["profit"],
"revenue": row["revenue"]
}
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/languages/{date_range}_languages_dataset.json",
"w") as file:
json.dump(languages, file)
except requests.exceptions.RequestException as e:
print("Failed - create_languages_dataset()")
print(e)
send_email(
"*****@*****.**",
"Failed - create_languages_dataset() at " +
str(datetime.now().strftime("%Y-%m-%d %H:%M")), e)
sys.exit()
def create_countries_dataset(token, start_date, end_date, date_range):
try:
url = f"https://api.voluum.com/report?from={start_date}T00:00:00Z&to={end_date}T00:00:00Z&tz=America%2FLos_Angeles&conversionTimeMode=VISIT&sort=campaignName&direction=asc&columns=campaignName&columns=countryName&columns=campaignId&columns=visits&columns=conversions&columns=revenue&columns=cost&columns=profit&columns=cpv&columns=cv&columns=roi&columns=epv&columns=cpa&groupBy=campaign&groupBy=country-code&offset=0&limit=100000&include=ACTIVE&filter1=traffic-source&filter1Value=37bbd390-ed90-4978-9066-09affa682bcc"
res = requests.get(url, headers={"cwauth-token": token})
res.raise_for_status()
res = res.json()
countries = {
"metadata": {
"vol_start_date": start_date,
"vol_end_date": end_date
},
"data": {}
}
campaigns = get_campaign_sets()
vol_ids = []
for campaign in campaigns:
vol_ids.append(campaign["vol_id"])
for row in res["rows"]:
country_name = row["countryName"]
campaign_id = row["campaignId"]
if campaign_id not in vol_ids:
continue
if country_name not in countries["data"]:
countries["data"][country_name] = {
campaign_id: {
"campaign_id": campaign_id,
"country_name": country_name,
"clicks": row["visits"],
"conversions": 0,
"leads": 0,
"sales": 0,
"cost": row["cost"],
"profit": row["profit"],
"revenue": row["revenue"]
}
}
else:
countries["data"][country_name][campaign_id] = {
"campaign_id": campaign_id,
"country_name": country_name,
"clicks": row["visits"],
"conversions": 0,
"leads": 0,
"sales": 0,
"cost": row["cost"],
"profit": row["profit"],
"revenue": row["revenue"]
}
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/conversions_for_each_campaign/{date_range}_conversions_for_each_campaign_dataset.json',
'r') as file:
json_file = json.load(file)
conversions_for_each_campaign = json_file["data"]
for campaign_id in conversions_for_each_campaign:
if (campaign_id not in vol_ids):
continue
for conversion in conversions_for_each_campaign[campaign_id]:
country_name = conversion["countryName"]
if country_name in countries["data"]:
countries["data"][country_name][campaign_id][
"conversions"] += 1
if conversion["transactionId"] == "account":
countries["data"][country_name][campaign_id][
"leads"] += 1
elif conversion["transactionId"] == "deposit":
countries["data"][country_name][campaign_id][
"sales"] += 1
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/countries/{date_range}_countries_dataset.json",
"w") as file:
json.dump(countries, file)
except requests.exceptions.RequestException as e:
print("Failed - create_countries_dataset()")
print(e)
send_email(
"*****@*****.**",
"Failed - create_countries_dataset() at " +
str(datetime.now().strftime("%Y-%m-%d %H:%M")), e)
sys.exit()
def create_campaigns_for_all_campaigns_dataset(vol_token, mgid_token, days_ago,
output_name):
try:
vol_dates = create_vol_date_range(days_ago, mgid_timezone)
vol_start_date = vol_dates[0]
vol_end_date = vol_dates[1]
mgid_dates = create_mgid_date_range(days_ago, mgid_timezone)
mgid_start_date = mgid_dates[0]
mgid_end_date = mgid_dates[1]
# get leads, sales, and revenue by campaign from voluum
vol_campaign_data = get_all_campaign_conversions_by_traffic_source(
vol_token, mgidVolTrafficSourceId, vol_start_date, vol_end_date)
# get clicks, imps, and cost by campaign from mgid
mgid_campaign_data = get_mgid_campaign_costs(
mgid_token, mgid_client_id, mgid_start_date,
mgid_end_date)["campaigns-stat"]
# get campaign_sets
campaign_sets = get_campaign_sets()
# create an dictionary to hold data and metadata
# the metadata is for request dates
# the data is an array of dicts where each dict is a campaign
# the stats in each campaign come from both mgid and voluum
# the campaign data is only collected for campains in campaign_sets.txt
campaigns_data = {
"metadata": {
"mgid_start_date": mgid_start_date,
"mgid_end_date": mgid_end_date,
"vol_start_date": vol_start_date,
"vol_end_date": vol_end_date,
},
"data": []
}
pattern = re.compile(r'.*cpc_(.*)')
for row in campaign_sets:
# extract the data out of a single campaign
mgid_campaign_id = row["mgid_id"]
vol_campaign_id = row["vol_id"]
campaign_name = row["name"]
pattern = re.compile(r'\d+.\d+')
res = pattern.findall(campaign_name)
bid = float(list(res)[0])
max_lead_cpa = row["max_lead_cpa"]
max_sale_cpa = row["max_sale_cpa"]
res = pattern.findall(campaign_name)
max_cpc = list(res)[0]
# create an empty dict to hold the final data for a single campaign
campaign_data = {}
# fill in the single campaign data
# some data comes from campaigns_sets.txt
# some data comes from mgid
# some data comes from vol
campaign_data["mgid_id"] = mgid_campaign_id
campaign_data["vol_id"] = vol_campaign_id
campaign_data["name"] = campaign_name
campaign_data["bid"] = bid
campaign_data["max_lead_cpa"] = max_lead_cpa
campaign_data["max_sale_cpa"] = max_sale_cpa
campaign_data["max_cpc"] = max_cpc
campaign_data["clicks"] = mgid_campaign_data[mgid_campaign_id][
"clicks"]
campaign_data["imps"] = mgid_campaign_data[mgid_campaign_id][
"imps"]
campaign_data["cost"] = mgid_campaign_data[mgid_campaign_id][
"spent"]
if vol_campaign_id in vol_campaign_data:
campaign_data["leads"] = vol_campaign_data[vol_campaign_id][
"leads"]
campaign_data["sales"] = vol_campaign_data[vol_campaign_id][
"sales"]
campaign_data["revenue"] = vol_campaign_data[vol_campaign_id][
"revenue"]
else:
campaign_data["leads"] = 0
campaign_data["sales"] = 0
campaign_data["revenue"] = 0
campaigns_data["data"].append(campaign_data)
for campaign in campaigns_data["data"]:
campaign["classification"] = classify_campaign_for_all_campaigns(
campaign)
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/campaigns_for_all_campaigns/{output_name}.json",
"w") as file:
json.dump(campaigns_data, file)
print(f"{output_name} created")
except Exception as e:
print("Failed - email sent")
send_email("*****@*****.**",
"Failed - create_campaigns_for_all_campaigns_dataset()", e)
sys.exit()
def create_complete_c_widgets_dataset(date_range, output_name):
try:
# 1. get some prerequisite data
campaigns = get_campaign_sets()
widget_whitelist = get_whitelist()
widget_greylist = get_greylist()
widget_blacklist = get_blacklist()
# these dates are only used for determining status = included.
# that is why they are hard coded to 90 days
mgid_dates = create_mgid_date_range(90, mgid_timezone)
mgid_start_date = mgid_dates[0]
mgid_end_date = mgid_dates[1]
########################################################
# 2. set up the basic data structure you want to create
complete_c_widgets = {}
#########################################################
# 3. create the "for_all_campaigns" part of each c widget
for campaign in campaigns:
vol_id = campaign["vol_id"]
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{vol_id}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json',
'r') as file:
json_file = json.load(file)
c_widget_pattern = re.compile(r'.*s.*')
p_widget_pattern = re.compile(r'\d*')
for widget in json_file["data"]:
c_widget = c_widget_pattern.search(widget)
if c_widget == None:
continue
c_widget = c_widget.group()
p_widget = p_widget_pattern.search(widget).group()
if c_widget in complete_c_widgets:
complete_c_widgets[c_widget]["for_all_campaigns"][
"clicks"] += json_file["data"][widget]["clicks"]
complete_c_widgets[c_widget]["for_all_campaigns"][
"cost"] += json_file["data"][widget]["cost"]
complete_c_widgets[c_widget]["for_all_campaigns"][
"revenue"] += json_file["data"][widget]["revenue"]
complete_c_widgets[c_widget]["for_all_campaigns"][
"leads"] += json_file["data"][widget]["leads"]
complete_c_widgets[c_widget]["for_all_campaigns"][
"sales"] += json_file["data"][widget]["sales"]
else:
complete_c_widgets[c_widget] = {"for_all_campaigns": {}}
complete_c_widgets[c_widget][
"for_all_campaigns"] = json_file["data"][widget]
complete_c_widgets[c_widget]["for_all_campaigns"][
"widget_id"] = c_widget
if p_widget in widget_whitelist:
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "p_whitelist"
elif p_widget in widget_greylist:
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "p_greylist"
elif p_widget in widget_blacklist:
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "p_blacklist"
if c_widget in widget_whitelist:
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "c_whitelist"
elif c_widget in widget_greylist:
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "c_greylist"
elif c_widget in widget_blacklist:
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "c_blacklist"
if (c_widget in widget_whitelist) & (p_widget
in widget_whitelist):
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "pc_whitelist"
elif (c_widget in widget_greylist) & (p_widget
in widget_greylist):
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "pc_greylist"
elif (c_widget in widget_blacklist) & (
p_widget in widget_blacklist):
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "pc_blacklist"
if (c_widget not in widget_whitelist) & (
p_widget not in widget_whitelist) & (
c_widget not in widget_greylist) & (
p_widget not in widget_greylist) & (
c_widget not in widget_blacklist) & (
p_widget not in widget_blacklist):
complete_c_widgets[c_widget]["for_all_campaigns"][
'global_status'] = "waiting"
#########################################################
# 5.
for c_widget in complete_c_widgets:
complete_c_widgets[c_widget]["for_each_campaign"] = []
for campaign in campaigns:
vol_id = campaign["vol_id"]
mgid_id = campaign["mgid_id"]
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{vol_id}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json',
'r') as file:
json_file = json.load(file)
included_widgets = get_mgid_included_widgets_by_campaign(
mgid_token, mgid_id, mgid_start_date, mgid_end_date)
excluded_widgets = get_mgid_excluded_widgets_by_campaign(
mgid_token, mgid_client_id, mgid_id)
c_widgets_for_one_campaign = {}
c_widget_pattern = re.compile(r'.*s.*')
p_widget_pattern = re.compile(r'\d*')
# mpc_pattern = re.compile(r'.*cpc_(.*)')
for widget in json_file["data"]:
c_widget = c_widget_pattern.search(widget)
if c_widget == None:
continue
c_widget = c_widget.group()
p_widget = p_widget_pattern.search(widget).group()
if c_widget in c_widgets_for_one_campaign:
c_widgets_for_one_campaign[c_widget][
"clicks"] += json_file["data"][widget]["clicks"]
c_widgets_for_one_campaign[c_widget]["cost"] += json_file[
"data"][widget]["cost"]
c_widgets_for_one_campaign[c_widget][
"revenue"] += json_file["data"][widget]["revenue"]
c_widgets_for_one_campaign[c_widget]["leads"] += json_file[
"data"][widget]["leads"]
c_widgets_for_one_campaign[c_widget]["sales"] += json_file[
"data"][widget]["sales"]
else:
c_widgets_for_one_campaign[c_widget] = json_file["data"][
widget]
if c_widget in excluded_widgets or p_widget in excluded_widgets:
c_widgets_for_one_campaign[c_widget][
'status'] = "excluded"
elif c_widget in included_widgets or p_widget in included_widgets:
c_widgets_for_one_campaign[c_widget][
'status'] = "included"
else:
c_widgets_for_one_campaign[c_widget][
'status'] = "inactive"
c_widgets_for_one_campaign[c_widget]["vol_id"] = campaign[
"vol_id"]
c_widgets_for_one_campaign[c_widget]["mgid_id"] = campaign[
"mgid_id"]
c_widgets_for_one_campaign[c_widget]["name"] = campaign[
"name"]
c_widgets_for_one_campaign[c_widget]["mpl"] = campaign[
"max_lead_cpa"]
c_widgets_for_one_campaign[c_widget]["mps"] = campaign[
"max_sale_cpa"]
list_to_get_c_bid = campaign["name"].split('_')
c_bid = float(list_to_get_c_bid[len(list_to_get_c_bid) -
1])
c_widgets_for_one_campaign[c_widget]["c_bid"] = c_bid
# res = mpc_pattern.findall(campaign["name"])
# c_widgets_for_one_campaign[c_widget]["c_bid"] = float(list(res)[0])
c_widgets_for_one_campaign[c_widget][
"w_bid"] = c_widgets_for_one_campaign[c_widget][
"c_bid"] * c_widgets_for_one_campaign[c_widget][
"coeff"]
for c_widget in c_widgets_for_one_campaign:
if complete_c_widgets[c_widget]["for_each_campaign"]:
complete_c_widgets[c_widget]["for_each_campaign"].append(
c_widgets_for_one_campaign[c_widget])
else:
complete_c_widgets[c_widget]["for_each_campaign"] = [
c_widgets_for_one_campaign[c_widget]
]
#################################################################33
# 5. Create the "campaign_counts" part of each c widget
for c_widget in complete_c_widgets:
complete_c_widgets[c_widget]["good_campaigns_count"] = 0
complete_c_widgets[c_widget]["bad_campaigns_count"] = 0
complete_c_widgets[c_widget]["wait_campaigns_count"] = 0
for c_widget in complete_c_widgets:
total_sales = complete_c_widgets[c_widget]["for_all_campaigns"][
"sales"]
complete_c_widgets[c_widget]["for_all_campaigns"][
"has_bad_and_included_campaigns"] = False
for campaign in complete_c_widgets[c_widget]["for_each_campaign"]:
# This is where each campaign is classified and the good/bad/wait counts are recorded
classification = classify_campaign_for_one_p_or_c_widget(
campaign, total_sales)
campaign["classification"] = classification
if classification == "good":
complete_c_widgets[c_widget]["good_campaigns_count"] += 1
elif classification == "half good":
complete_c_widgets[c_widget]["good_campaigns_count"] += .5
elif classification == "bad":
complete_c_widgets[c_widget]["bad_campaigns_count"] += 1
if campaign["status"] == "included":
complete_c_widgets[c_widget]["for_all_campaigns"][
"has_bad_and_included_campaigns"] = True
elif classification == "half bad":
complete_c_widgets[c_widget]["bad_campaigns_count"] += .5
elif classification == "wait":
complete_c_widgets[c_widget]["wait_campaigns_count"] += 1
#############################################################
# 6. create isBadAndIncluded variable for each campaign
for c_widget in complete_c_widgets:
for campaign in complete_c_widgets[c_widget]["for_each_campaign"]:
if (campaign["classification"] == "bad") & (campaign["status"]
== "included"):
campaign["is_bad_and_included"] = True
else:
campaign["is_bad_and_included"] = False
#############################################################
# 7. Classify the c widget
for c_widget in complete_c_widgets.values():
c_widget["for_all_campaigns"][
"classification"] = classify_c_widget_for_all_campaigns(
c_widget)
#############################################################
# 8. create hasMismatchClassificationAndGlobalStatus variable in
# "for_all_campaigns"
for c_widget in complete_c_widgets.values():
if (c_widget['for_all_campaigns']['classification'] != "wait") & (
(c_widget["for_all_campaigns"]["global_status"] !=
f"p_{c_widget['for_all_campaigns']['classification']}list") &
(c_widget["for_all_campaigns"]["global_status"] !=
f"c_{c_widget['for_all_campaigns']['classification']}list") &
(c_widget["for_all_campaigns"]["global_status"] !=
f"pc_{c_widget['for_all_campaigns']['classification']}list")):
c_widget["for_all_campaigns"][
"has_mismatch_classification_and_global_status"] = True
else:
c_widget["for_all_campaigns"][
"has_mismatch_classification_and_global_status"] = False
###########################################################
# add domains, if possible
with open(
f'{os.environ.get("ULANMEDIAAPP")}/curated_lists/widget_domains/widget_domains.json',
'r') as file:
widget_domains_lookup = json.load(file)
p_widget_pattern = re.compile(r'\d*')
for c_widget in complete_c_widgets:
p_widget = p_widget_pattern.search(c_widget).group()
if p_widget in widget_domains_lookup:
complete_c_widgets[c_widget]["for_all_campaigns"][
"domain"] = ""
for domain in widget_domains_lookup[p_widget]["domains"]:
if complete_c_widgets[c_widget]["for_all_campaigns"][
"domain"] == "":
complete_c_widgets[c_widget]["for_all_campaigns"][
"domain"] = domain
else:
complete_c_widgets[c_widget]["for_all_campaigns"][
"domain"] = complete_c_widgets[c_widget][
"for_all_campaigns"]["domain"] + "," + domain
else:
complete_c_widgets[c_widget]["for_all_campaigns"][
"domain"] = ""
############################################################
# 9. Save complete_p_widgets to a json file and return it as a
# json file
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/complete_c_widgets/{output_name}.json",
"w") as file:
json.dump(complete_c_widgets, file)
return complete_c_widgets
except:
print("Failed - email sent")
send_email("*****@*****.**",
"Failed - create_complete_c_widgets_dataset()",
"Failed - create_complete_c_widgets_dataset()")
sys.exit()
def add_yesterday_in_days_for_one_campaign_dataset(vol_token, mgid_token):
try:
# create campaigns lookup dictionary
campaigns_sets = get_campaign_sets()
campaigns_lookup = {}
for campaign in campaigns_sets:
campaigns_lookup[campaign["vol_id"]] = campaign["mgid_id"]
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/days_for_one_campaign/days_for_one_campaign_dataset.json', 'r') as file:
days_for_one_campaign_dataset = json.load(file)
# create vol dates
# to get yesterday's daily stats, vol_start_date is yesterday and
# vol_end_date is today
vol_date_range = create_vol_date_range(1, mgid_timezone)
vol_start_date = vol_date_range[0]
vol_end_date = vol_date_range[1]
# create mgid dates
# to get yesterday's daily stats, mgid_start_date is yesterday and
# mgid_end_date is yesterday
mgid_date_range = create_mgid_date_range(1, mgid_timezone)
mgid_start_date = mgid_date_range[0]
mgid_end_date = mgid_date_range[1]
# set up the dictionary to store daily stats by campaign
# keys are vol_id, values are yesterday's daily stats for that campaign
daily_stats= {}
for campaign in campaigns_sets:
daily_stats[campaign["vol_id"]] = []
# get vol and mgid data
vol_response = get_vol_daily_stats_data(vol_token, vol_start_date,
vol_end_date, mgid_timezone)
mgid_response = get_mgid_daily_stats_data(mgid_token, mgid_start_date,
mgid_end_date)
# fill in the daily_stats dictionary
# each key is a vol campaign id
# each value is a dictionary of data, some from mgid, some from vol
# data from mgid
# clicks
# cost
# data from vol
# conversions
# revenue
# name
# day
# Some of this code is complicated and going to be very difficult to
# understand. The reason for all the loops and conditionals is that if
# a campaign does not have any data for a particular day, voluum
# returns nothing rather than returning a response that says clicks=0,
# etc. That is why I had to create lists of campaigns with data and
# campaigns without data.
campaigns_with_day_data = []
campaigns_without_day_data = []
day = ""
for campaign in vol_response:
vol_id = campaign["campaignId"]
day = campaign["day"]
campaigns_with_day_data.append(vol_id)
# this conditional will exclude vol responses from campaigns that
# are no longer in campaign_sets.txt
if vol_id in daily_stats:
mgid_id = str(campaigns_lookup[vol_id])
daily_stats[vol_id].append({"vol_id": vol_id,
"conversions": campaign["conversions"],
"revenue": campaign["revenue"],
"name": re.sub(r"^.* - ", "",campaign["campaignName"], count=1),
"day": campaign["day"],
"clicks": mgid_response[mgid_id]["clicks"],
"cost": mgid_response[mgid_id]["spent"],
})
for campaign in daily_stats.keys():
if campaign not in campaigns_with_day_data:
campaigns_without_day_data.append(campaign)
for campaign in campaigns_without_day_data:
daily_stats[campaign].append({"vol_id": campaign,
"conversions": 0,
"revenue": 0,
"name": "",
"day": day,
"clicks": 0,
"cost": 0,
})
for campaign in days_for_one_campaign_dataset:
# add daily stats for yesterday
days_for_one_campaign_dataset[campaign].insert(0,
daily_stats[campaign][0] )
# remove daily stats for 50 days ago
days_for_one_campaign_dataset[campaign].pop()
# create a json file
with open(f"../../data/days_for_one_campaign/days_for_one_campaign_dataset.json", "w") as file:
json.dump(days_for_one_campaign_dataset, file)
print(f"yesterday's daily stats added to days_for_one_campaign_dataset.json" )
except:
print("Failed - email sent")
send_email("*****@*****.**", "Failed - add_yesterday_in_days_for_one_campaign_dataset()", "Failed - add_yesterday_in_days_for_one_campaign_dataset()")
sys.exit()
def create_days_for_one_campaign_dataset(vol_token, mgid_token, days_ago,
output_name):
try:
# create campaigns lookup dictionary so that you can find the corresponding
# mgid_id from the vol_id
# keys are vol_id, values are mgid_id
campaigns_sets = get_campaign_sets()
campaigns_lookup = {}
for campaign in campaigns_sets:
campaigns_lookup[campaign["vol_id"]] = campaign["mgid_id"]
# set up the dictionary to store daily stats by campaign
# keys are vol_id, values are empty lists that will be filled
# with a dictionaries of daily stats
daily_stats = {}
for campaign in campaigns_sets:
daily_stats[campaign["vol_id"]] = []
# fill the dictionary with data from each day
for i in range(1, days_ago + 1):
# create vol dates
# to get yesterday's daily stats, vol_start_date is yesterday and
# vol_end_date is today
start_date_utc = pytz.utc.localize(
datetime.utcnow()) - timedelta(i)
start_date_pst = start_date_utc.astimezone(
pytz.timezone(mgid_timezone))
end_date_utc = pytz.utc.localize(datetime.utcnow() -
timedelta(i - 1))
end_date_pst = end_date_utc.astimezone(
pytz.timezone(mgid_timezone))
vol_start_date = start_date_pst.strftime("%Y-%m-%d")
vol_end_date = end_date_pst.strftime("%Y-%m-%d")
# create mgid dates
# to get yesterday's daily stats, mgid_start_date is yesterday and
# mgid_end_date is yesterday
start_date_utc = pytz.utc.localize(
datetime.utcnow()) - timedelta(i)
start_date_pst = start_date_utc.astimezone(
pytz.timezone(mgid_timezone))
mgid_start_date = start_date_pst.strftime("%Y-%m-%d")
mgid_end_date = mgid_start_date
# get vol and mgid data
vol_response = get_vol_daily_stats_data(vol_token, vol_start_date,
vol_end_date,
mgid_timezone)
mgid_response = get_mgid_daily_stats_data(mgid_token,
mgid_start_date,
mgid_end_date)
# fill in the daily_stats dictionary
# each key is a vol campaign id
# each value is a dictionary of data, some from mgid, some from vol
# data from mgid
# clicks
# cost
# data from vol
# conversions
# revenue
# name
# day
# Some of this code is complicated and going to be very difficult to
# understand. The reason for all the loops and conditionals is that if
# a campaign does not have any data for a particular day, voluum
# returns nothing rather than returning a response that says clicks=0,
# etc. That is why I had to create lists of campaigns with data and
# campaigns without data.
campaigns_with_day_data = []
campaigns_without_day_data = []
day = ""
for campaign in vol_response:
vol_id = campaign["campaignId"]
day = campaign["day"]
campaigns_with_day_data.append(vol_id)
# this conditional will exclude vol responses from campaigns that
# are no longer in campaign_sets.txt
if vol_id in daily_stats:
mgid_id = str(campaigns_lookup[vol_id])
daily_stats[vol_id].append({
"vol_id":
vol_id,
"conversions":
campaign["conversions"],
"revenue":
campaign["revenue"],
"name":
re.sub(r"^.* - ",
"",
campaign["campaignName"],
count=1),
"day":
campaign["day"],
"clicks":
mgid_response[mgid_id]["clicks"],
"cost":
mgid_response[mgid_id]["spent"],
})
for campaign in daily_stats.keys():
if campaign not in campaigns_with_day_data:
campaigns_without_day_data.append(campaign)
for campaign in campaigns_without_day_data:
daily_stats[campaign].append({
"vol_id": campaign,
"conversions": 0,
"revenue": 0,
"name": "",
"day": day,
"clicks": 0,
"cost": 0,
})
# create a json file
with open(f"../../data/days_for_one_campaign/{output_name}.json",
"w") as file:
json.dump(daily_stats, file)
print(f"{output_name} created")
except:
print("Failed - email sent")
send_email("*****@*****.**",
"Failed - create_days_for_one_campaign_dataset()",
"Failed - create_days_for_one_campaign_dataset()")
sys.exit()
def create_complete_p_widgets_dataset(date_range, output_name):
try:
# 1. get some prerequisite data
campaigns = get_campaign_sets()
widget_whitelist = get_whitelist()
widget_greylist = get_greylist()
widget_blacklist = get_blacklist()
# these dates are only used for determining status = included.
# that is why they are hard coded to 90 days
mgid_dates = create_mgid_date_range(90, mgid_timezone)
mgid_start_date = mgid_dates[0]
mgid_end_date = mgid_dates[1]
########################################################
# 2. set up the basic data structure you want to create
complete_p_widgets = {}
#########################################################
# 3. create the "for_all_campaigns" part of each p widget
for campaign in campaigns:
vol_id = campaign["vol_id"]
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{vol_id}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json', 'r') as file:
json_file = json.load(file)
pattern = re.compile(r'\d*')
for widget in json_file["data"]:
p_widget = pattern.search(widget).group()
if p_widget in complete_p_widgets:
complete_p_widgets[p_widget]["for_all_campaigns"]["clicks"] += json_file["data"][widget]["clicks"]
complete_p_widgets[p_widget]["for_all_campaigns"]["cost"] += json_file["data"][widget]["cost"]
complete_p_widgets[p_widget]["for_all_campaigns"]["revenue"] += json_file["data"][widget]["revenue"]
complete_p_widgets[p_widget]["for_all_campaigns"]["leads"] += json_file["data"][widget]["leads"]
complete_p_widgets[p_widget]["for_all_campaigns"]["sales"] += json_file["data"][widget]["sales"]
else:
complete_p_widgets[p_widget] = {"for_all_campaigns": {}}
complete_p_widgets[p_widget]["for_all_campaigns"] = json_file["data"][widget]
complete_p_widgets[p_widget]["for_all_campaigns"]["widget_id"] = p_widget
if p_widget in widget_whitelist:
complete_p_widgets[p_widget]["for_all_campaigns"]['global_status'] = "p_whitelist"
elif p_widget in widget_greylist:
complete_p_widgets[p_widget]["for_all_campaigns"]['global_status'] = "p_greylist"
elif p_widget in widget_blacklist:
complete_p_widgets[p_widget]["for_all_campaigns"]['global_status'] = "p_blacklist"
else:
complete_p_widgets[p_widget]["for_all_campaigns"]['global_status'] = "waiting"
if widget is not p_widget:
complete_p_widgets[p_widget]["for_all_campaigns"]["has_children"] = True
else:
complete_p_widgets[p_widget]["for_all_campaigns"]["has_children"] = False
#########################################################
# 4. create the "for_each_campaign" part of each p widget
for p_widget in complete_p_widgets:
complete_p_widgets[p_widget]["for_each_campaign"] = []
for campaign in campaigns:
vol_id = campaign["vol_id"]
mgid_id = campaign["mgid_id"]
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{vol_id}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json', 'r') as file:
json_file = json.load(file)
included_widgets = get_mgid_included_widgets_by_campaign(mgid_token,
mgid_id, mgid_start_date, mgid_end_date)
excluded_widgets = get_mgid_excluded_widgets_by_campaign(mgid_token, mgid_client_id, mgid_id)
p_widgets_for_one_campaign = {}
for widget in json_file["data"]:
p_widget = pattern.search(widget).group()
if p_widget in p_widgets_for_one_campaign:
p_widgets_for_one_campaign[p_widget]["clicks"] += json_file["data"][widget]["clicks"]
p_widgets_for_one_campaign[p_widget]["cost"] += json_file["data"][widget]["cost"]
p_widgets_for_one_campaign[p_widget]["revenue"] += json_file["data"][widget]["revenue"]
p_widgets_for_one_campaign[p_widget]["leads"] += json_file["data"][widget]["leads"]
p_widgets_for_one_campaign[p_widget]["sales"] += json_file["data"][widget]["sales"]
else:
p_widgets_for_one_campaign[p_widget] = json_file["data"][widget]
p_widgets_for_one_campaign[p_widget]["widget_id"] = p_widget
if p_widget in excluded_widgets:
p_widgets_for_one_campaign[p_widget]['status'] = "excluded"
elif p_widget in included_widgets:
p_widgets_for_one_campaign[p_widget]['status'] = "included"
else:
p_widgets_for_one_campaign[p_widget]['status'] = "inactive"
#############
# hardcoded inactive
# this is for when the mgid api shows clicks for a campaign
# widget, but the mgid web dashboard doesn't show any clicks for
# the same campaign widget, even though the date range is the
# same.
# These should manually removed after enough time has passed
# for the api to no longer show any clicks for the campaign
# widget
# these were removed on 8/23/19
# hardcoded_campaigns = ["506299", "506320", "506244", "506323"]
# hardcoded_p_widgets = ["5722923"]
# if p_widgets_for_one_campaign[p_widget]["widget_id"] in hardcoded_p_widgets and mgid_id in hardcoded_campaigns:
# p_widgets_for_one_campaign[p_widget]['status'] = "inactive"
############
############
############
p_widgets_for_one_campaign[p_widget]["vol_id"] = campaign["vol_id"]
p_widgets_for_one_campaign[p_widget]["mgid_id"] = campaign["mgid_id"]
p_widgets_for_one_campaign[p_widget]["name"] = campaign["name"]
p_widgets_for_one_campaign[p_widget]["mpl"] = campaign["max_lead_cpa"]
p_widgets_for_one_campaign[p_widget]["mps"] = campaign["max_sale_cpa"]
list_to_get_c_bid = campaign["name"].split('_')
c_bid = float(list_to_get_c_bid[len(list_to_get_c_bid) - 1])
p_widgets_for_one_campaign[p_widget]["c_bid"] = c_bid
p_widgets_for_one_campaign[p_widget]["w_bid"] = p_widgets_for_one_campaign[p_widget]["c_bid"] * p_widgets_for_one_campaign[p_widget]["coeff"]
for p_widget in p_widgets_for_one_campaign:
if complete_p_widgets[p_widget]["for_each_campaign"]:
complete_p_widgets[p_widget]["for_each_campaign"].append(p_widgets_for_one_campaign[p_widget])
else:
complete_p_widgets[p_widget]["for_each_campaign"] = [p_widgets_for_one_campaign[p_widget]]
#################################################################33
# 5. Create the "campaign_counts" part of each p widget
for p_widget in complete_p_widgets:
complete_p_widgets[p_widget]["good_campaigns_count"] = 0
complete_p_widgets[p_widget]["bad_campaigns_count"] = 0
complete_p_widgets[p_widget]["wait_campaigns_count"] = 0
for p_widget in complete_p_widgets:
total_sales = complete_p_widgets[p_widget]["for_all_campaigns"]["sales"]
complete_p_widgets[p_widget]["for_all_campaigns"]["has_bad_and_included_campaigns"] = False
for campaign in complete_p_widgets[p_widget]["for_each_campaign"]:
# This is where each campaign is classified and the good/bad/not
# yet
# counts are recorded
classification = classify_campaign_for_one_p_or_c_widget(campaign, total_sales)
campaign["classification"] = classification
if classification == "good":
complete_p_widgets[p_widget]["good_campaigns_count"] += 1
elif classification == "half good":
complete_p_widgets[p_widget]["good_campaigns_count"] += .5
elif classification == "bad":
complete_p_widgets[p_widget]["bad_campaigns_count"] += 1
if campaign["status"] == "included":
complete_p_widgets[p_widget]["for_all_campaigns"]["has_bad_and_included_campaigns"] = True
elif classification == "half bad":
complete_p_widgets[p_widget]["bad_campaigns_count"] += .5
elif classification == "wait":
complete_p_widgets[p_widget]["wait_campaigns_count"] += 1
#############################################################
# 6. create is_bad_and_included variable for each campaign
for p_widget in complete_p_widgets:
for campaign in complete_p_widgets[p_widget]["for_each_campaign"]:
if (campaign["classification"] == "bad") & (campaign["status"] ==
"included"):
campaign["is_bad_and_included"] = True
else:
campaign["is_bad_and_included"] = False
#############################################################
# 7. create the classification of each p widget
for p_widget in complete_p_widgets.values():
p_widget["for_all_campaigns"]["classification"] = classify_p_widget_for_all_campaigns(p_widget)
#############################################################
# 8. create has_mismatch_classification_and_global_status variable in
# "for_all_campaigns"
for p_widget in complete_p_widgets.values():
if (p_widget['for_all_campaigns']['classification'] != "wait") & (p_widget["for_all_campaigns"]["global_status"] != f"p_{p_widget['for_all_campaigns']['classification']}list"):
p_widget["for_all_campaigns"]["has_mismatch_classification_and_global_status"] = True
else:
p_widget["for_all_campaigns"]["has_mismatch_classification_and_global_status"] = False
###########################################################
# add domains, if possible
with open(f'{os.environ.get("ULANMEDIAAPP")}/curated_lists/widget_domains/widget_domains.json', 'r') as file:
widget_domains_lookup = json.load(file)
for p_widget in complete_p_widgets:
if p_widget in widget_domains_lookup:
complete_p_widgets[p_widget]["for_all_campaigns"]["domain"] = ""
for domain in widget_domains_lookup[p_widget]["domains"]:
if complete_p_widgets[p_widget]["for_all_campaigns"]["domain"] == "":
complete_p_widgets[p_widget]["for_all_campaigns"]["domain"] = domain
else:
complete_p_widgets[p_widget]["for_all_campaigns"]["domain"] = complete_p_widgets[p_widget]["for_all_campaigns"]["domain"] + "," + domain
else:
complete_p_widgets[p_widget]["for_all_campaigns"]["domain"] = ""
#############################################################
# 9. Save complete_p_widgets to a json file and return it as a
# json file
with open(f"{os.environ.get('ULANMEDIAAPP')}/data/complete_p_widgets/{output_name}.json", "w") as file:
json.dump(complete_p_widgets, file)
return complete_p_widgets
except Exception as e:
print("Failed - email sent")
send_email("*****@*****.**", "Failed - create_complete_p_widgets_dataset()", e)
sys.exit()
def create_campaigns_for_one_p_widget_dataset(p_widget_id, date_range,
max_rec_bid):
try:
########################################################
# 1. get some prerequisite data
max_rec_bid = float(max_rec_bid)
campaigns = get_campaign_sets()
widget_whitelist = get_whitelist()
widget_greylist = get_greylist()
widget_blacklist = get_blacklist()
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/complete_p_widgets/{date_range}_complete_p_widgets_dataset.json', 'r') as file:
complete_p_widgets = json.load(file)
if complete_p_widgets.get(p_widget_id):
complete_p_widget = complete_p_widgets[p_widget_id]
else:
complete_p_widget = {"for_all_campaigns":{"domain": "unknown"}, "for_each_campaign":[]}
########################################################
# 2. set up the basic data structure you want to create
campaigns_for_one_p_widget = {"metadata":{}, "data":[]}
#########################################################
# 3. Add the metadata.
vol_id_for_adding_metadata = campaigns[0]["vol_id"]
with open(f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{campaigns[0]["vol_id"]}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json', 'r') as file:
json_file = json.load(file)
campaigns_for_one_p_widget["metadata"]["mgid_start_date"] = json_file["metadata"]["mgid_start_date"]
campaigns_for_one_p_widget["metadata"]["mgid_end_date"] = json_file["metadata"]["mgid_end_date"]
campaigns_for_one_p_widget["metadata"]["vol_start_date"] = json_file["metadata"]["vol_start_date"]
campaigns_for_one_p_widget["metadata"]["vol_end_date"] = json_file["metadata"]["vol_end_date"]
if complete_p_widgets.get(p_widget_id):
campaigns_for_one_p_widget["metadata"]["p_widget_classification"] = complete_p_widget["for_all_campaigns"]["classification"]
campaigns_for_one_p_widget["metadata"]["p_widget_global_status"] = complete_p_widget["for_all_campaigns"]["global_status"]
campaigns_for_one_p_widget["metadata"]["p_widget_has_children"] = complete_p_widget["for_all_campaigns"]["has_children"]
campaigns_for_one_p_widget["metadata"]["p_widget_has_mismatch_classification_and_global_status"] = complete_p_widget["for_all_campaigns"]["has_mismatch_classification_and_global_status"]
campaigns_for_one_p_widget["metadata"]["good_campaigns_count"] = complete_p_widget["good_campaigns_count"]
campaigns_for_one_p_widget["metadata"]["bad_campaigns_count"] = complete_p_widget["bad_campaigns_count"]
campaigns_for_one_p_widget["metadata"]["wait_campaigns_count"] = complete_p_widget["wait_campaigns_count"]
#########################################################
# 4. Add the data
campaigns_for_one_p_widget["data"] = complete_p_widget["for_each_campaign"]
for campaign in campaigns_for_one_p_widget["data"]:
campaign["domain"] = complete_p_widget["for_all_campaigns"]["domain"]
for campaign in campaigns_for_one_p_widget["data"]:
sales = campaign["sales"]
mpl = campaign["mpl"]
if campaign["leads"] > 0:
cpl = campaign["cost"]/campaign["leads"]
if campaign["clicks"] > 0:
epc = campaign["revenue"]/campaign["clicks"]
c_bid = campaign["c_bid"]
w_bid = campaign["w_bid"]
coeff = campaign["coeff"]
if sales > 0:
campaign["rec_w_bid"] = epc - epc * .3
elif campaign["leads"] > 0:
if cpl == 0:
campaign["rec_w_bid"] = c_bid
else:
campaign["rec_w_bid"] = c_bid * mpl / cpl / 2
else:
campaign["rec_w_bid"] = c_bid
if campaign["rec_w_bid"] > max_rec_bid:
campaign["rec_w_bid"] = max_rec_bid
campaign["rec_coeff"] = campaign["rec_w_bid"] / c_bid
rec_w_bid = campaign["rec_w_bid"]
rec_coeff = campaign["rec_coeff"]
if w_bid != rec_w_bid:
campaign["mismatch_w_bid_and_rec_w_bid"] = True
else:
campaign["mismatch_w_bid_and_rec_w_bid"] = False
if coeff != rec_coeff:
campaign["mismatch_coeff_and_rec_coeff"] = True
else:
campaign["mismatch_coeff_and_rec_coeff"] = False
############################################################
# 11. Save campaigns_for_one_p_widget to a json file and return it as a
# json file
with open(f"{os.environ.get('ULANMEDIAAPP')}/data/campaigns_for_one_p_widget/{p_widget_id}_{date_range}_campaigns_for_one_p_widget_dataset.json", "w") as file:
json.dump(campaigns_for_one_p_widget, file)
return json.dumps(campaigns_for_one_p_widget)
except:
print("Failed - email sent")
send_email("*****@*****.**", "Failed - create_campaigns_for_one_p_widget_dataset()", "Failed - create_campaigns_for_one_p_widget_dataset()")
sys.exit()
def create_c_widgets_for_one_p_widget_dataset(p_widget, date_range):
try:
# This function creates a c_widgets_for_one_p_widget json data structure.
# That means each row is a c widget for a given p widget. The final data
# structure looks like this:
# {"metadata": {"mgid_start_date": "2018-07-05",
# "mgid_end_date": "2018-12-31",
# "vol_start_date": "2018-07-05",
# "vol_end_date": "2019-01-01"},
# "data ": {"5633835s713950": {"widget_id": "5633835s713950",
# "clicks": 20,
# "cost": 1.06,
# "revenue": 0.0,
# "leads": 0,
# "sales": 0,
# "referrer": [],
# "vol_id": "39eded96-4619-4c00-b0e6-bfdc622c894c",
# "mgid_id": "39eded96-4619-4c00-b0e6-bfdc622c894c",
# "max_lead_cpa": 5,
# "max_sale_cpa": 150},
# ...
# ...
# ...
########################################################
# 1. get some prerequisite data
campaigns = get_campaign_sets()
widget_whitelist = get_whitelist()
widget_greylist = get_greylist()
widget_blacklist = get_blacklist()
########################################################
# 2. set up the basic data structure you want to create
c_widgets_for_one_p_widget = {
"metadata": {
"mgid_start_date": "",
"mgid_end_date": "",
"vol_start_date": "",
"vol_end_date": ""
},
"data": {}
}
########################################################
# 3. Add the metadata. The metadata are the date ranges of the mgid and vol
# request dates. All p_and_c_widgets_for_one_campaign files have the same
# date ranges so I am just using the first campaign.
vol_id_for_adding_metadata = campaigns[0]["vol_id"]
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{campaigns[0]["vol_id"]}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json',
'r') as file:
json_file = json.load(file)
c_widgets_for_one_p_widget["metadata"]["mgid_start_date"] = json_file[
"metadata"]["mgid_start_date"]
c_widgets_for_one_p_widget["metadata"]["mgid_end_date"] = json_file[
"metadata"]["mgid_end_date"]
c_widgets_for_one_p_widget["metadata"]["vol_start_date"] = json_file[
"metadata"]["vol_start_date"]
c_widgets_for_one_p_widget["metadata"]["vol_end_date"] = json_file[
"metadata"]["vol_end_date"]
# At this point in the process, c_widgets_for_one_p_widget looks like this:
#{ 'metadata': { 'mgid_end_date': '2019-02-04',
# 'mgid_start_date': '2018-08-09',
# 'vol_end_date': '2019-02-05',
# 'vol_start_date': '2018-08-09'},
# 'data': {}
# }
########################################################
# 4. loop through p and c widgets for one campaign and create c widgets for
# one p widget
for campaign in campaigns:
vol_id = campaign["vol_id"]
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/p_and_c_widgets_for_one_campaign/{vol_id}_{date_range}_p_and_c_widgets_for_one_campaign_dataset.json',
'r') as file:
json_file = json.load(file)
data = json_file["data"]
pattern = re.compile(r'\d*')
for widget in data:
extracted_p_widget = pattern.search(widget).group()
if extracted_p_widget != p_widget:
continue
if widget in c_widgets_for_one_p_widget["data"]:
c_widgets_for_one_p_widget["data"][widget][
"clicks"] += data[widget]["clicks"]
c_widgets_for_one_p_widget["data"][widget]["cost"] += data[
widget]["cost"]
c_widgets_for_one_p_widget["data"][widget][
"revenue"] += data[widget]["revenue"]
c_widgets_for_one_p_widget["data"][widget][
"leads"] += data[widget]["leads"]
c_widgets_for_one_p_widget["data"][widget][
"sales"] += data[widget]["sales"]
else:
c_widgets_for_one_p_widget["data"][widget] = data[widget]
c_widgets_for_one_p_widget["data"][widget][
"vol_id"] = campaign["vol_id"]
c_widgets_for_one_p_widget["data"][widget][
"mgid_id"] = campaign["mgid_id"]
c_widgets_for_one_p_widget["data"][widget][
"mpl"] = campaign["max_lead_cpa"]
c_widgets_for_one_p_widget["data"][widget][
"mps"] = campaign["max_sale_cpa"]
list_to_get_mpc = campaign["name"].split('_')
c_widgets_for_one_p_widget["data"][widget]["mpc"] = float(
list_to_get_mpc[len(list_to_get_mpc) - 1])
if p_widget in widget_whitelist:
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "p_whitelist"
elif p_widget in widget_greylist:
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "p_greylist"
elif p_widget in widget_blacklist:
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "p_blacklist"
if widget in widget_whitelist:
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "c_whitelist"
elif widget in widget_greylist:
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "c_greylist"
elif widget in widget_blacklist:
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "c_blacklist"
if (widget in widget_whitelist) & (p_widget
in widget_whitelist):
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "pc_whitelist"
elif (widget in widget_greylist) & (p_widget
in widget_greylist):
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "pc_greylist"
elif (widget in widget_blacklist) & (p_widget
in widget_blacklist):
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "pc_blacklist"
if (widget not in widget_whitelist) & (
p_widget not in widget_whitelist) & (
widget not in widget_greylist) & (
p_widget not in widget_greylist) & (
widget not in widget_blacklist) & (
p_widget not in widget_blacklist):
c_widgets_for_one_p_widget["data"][widget][
"global_status"] = "waiting"
# add the domain
with open(
f'{os.environ.get("ULANMEDIAAPP")}/data/complete_p_widgets/{date_range}_complete_p_widgets_dataset.json',
'r') as file:
complete_p_widgets = json.load(file)
domain = complete_p_widgets[p_widget]["for_all_campaigns"]["domain"]
for c_widget in c_widgets_for_one_p_widget["data"]:
c_widgets_for_one_p_widget["data"][c_widget]["domain"] = domain
############################################################
# 5. Save c_widgets_for_one_p_widget to a json file and return it as a
# json file
with open(
f"{os.environ.get('ULANMEDIAAPP')}/data/c_widgets_for_one_p_widget/{p_widget}_{date_range}_c_widgets_for_one_p_widget_dataset.json",
"w") as file:
json.dump(c_widgets_for_one_p_widget, file)
return json.dumps(c_widgets_for_one_p_widget)
except Exception as e:
print("Failed - email sent")
send_email("*****@*****.**",
"Failed - create_c_widgets_for_one_p_widget_dataset()", e)
sys.exit()