def dcm_api_list(endpoint):
accounts = set(get_rows("user", project.task['accounts']))
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'],
account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
for item in API_DCM(
project.task['auth'], iterate=True,
internal=is_superuser).call(endpoint).list(**kwargs).execute():
yield item
def get_profile_for_api(auth, account_id):
"""Return a DCM profile ID for the currently supplied credentials.
Bulletproofing:
https://developers.google.com/doubleclick-advertisers/v3.2/userProfiles/get
Handles cases of superuser, otherwise chooses the first matched profile.
Allows DCM jobs to only specify account ID, which makes jobs more portable
between accounts.
Args:
* auth: (string) Either user or service.
* account_id: (int) Account number for which report is retrieved.
Returns:
* Is Superuser ( bool ): True if superuser account
* Profile ID ( int ): profile id to be used to make API call
Raises:
* If current credentials do not have a profile for this account.
"""
profile_admin = None
profile_network = None
for p in API_DCM(auth, iterate=True).userProfiles().list().execute():
p_id = int(p['profileId'])
a_id = int(p['accountId'])
# take the first profile for admin
if a_id == 2515 and 'subAccountId' not in p:
profile_admin = p_id
break
# try to find a network profile if exists
if a_id == account_id:
profile_network = p_id
break
if profile_admin:
return True, profile_admin
elif profile_network:
return False, profile_network
else:
raise Exception('Add your user profile to DCM account %s.' %
account_id)
def get_creatives(accounts):
if project.verbose: print 'DCM Ads'
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'],
account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
for ad in API_DCM(
"user", iterate=True,
internal=is_superuser).creatives().list(**kwargs).execute():
yield ad
def get_sites(accounts):
if project.verbose: print 'DCM Sites'
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'], account_id)
kwargs = { 'profileId':profile_id, 'accountId':account_id } if is_superuser else { 'profileId':profile_id }
for site in API_DCM("user", iterate=True, internal=is_superuser).sites().list(**kwargs).execute():
if long(site['accountId']) in accounts:
for contact in site.get('siteContacts', []):
SITE_CONTACTS.append([
site['accountId'],
site.get('subaccountId'),
site.get('directorySiteId'),
site['id'],
contact['id'],
contact['email'],
contact.get('firstName', ''),
contact.get('lastName', ''),
contact.get('title', ''),
contact.get('address', ''),
contact.get('phone', ''),
contact['contactType'],
])
yield [
site['accountId'],
site.get('subaccountId'),
site.get('directorySiteId'),
site['id'],
site['name'],
site['keyName'],
site['approved'],
site.get('orientation'),
site['siteSettings'].get('disableNewCookie'),
site['siteSettings'].get('activeViewOptOut'),
site['siteSettings'].get('adBlockingOptOut'),
site['siteSettings'].get('videoActiveViewOptOutTemplate'),
site['siteSettings'].get('vpaidAdapterChoiceTemplate'),
site['siteSettings'].get('tagSetting', {}).get('additionalKeyValues'),
site['siteSettings'].get('tagSetting', {}).get('includeClickTracking'),
site['siteSettings'].get('tagSetting', {}).get('includeClickThroughUrls'),
site['siteSettings'].get('tagSetting', {}).get('keywordOption'),
]
def cm_profile_load(config, task):
cm_profile_clear(config, task)
# write accounts to BQ
put_rows(
config, task['auth_bigquery'], {
'bigquery': {
'dataset':
task['dataset'],
'table':
'CM_Profiles',
'schema':
Discovery_To_BigQuery('dfareporting', 'v3.4').method_schema(
'userProfiles.list', iterate=True),
'format':
'JSON'
}
},
API_DCM(config, task['auth_cm'],
iterate=True).userProfiles().list().execute())
# write accounts to sheet
put_rows(
config, task['auth_sheets'], {
'sheets': {
'sheet': task['sheet'],
'tab': 'CM Profiles',
'header': False,
'range': 'B2'
}
},
get_rows(
config, task['auth_bigquery'], {
'bigquery': {
'dataset':
task['dataset'],
'query':
"SELECT CONCAT(accountName, ' - ', accountId), CONCAT(subAccountName, ' - ', subAccountId), profileId, userName FROM `%s.CM_Profiles`"
% task['dataset'],
'legacy':
False
}
}))
def load_multiple():
campaigns = [
str(lookup_id(r)) for r in set(
get_rows(config,
task['auth_cm'], {
'sheets': {
'sheet': task['sheet'],
'tab': 'CM Campaigns',
'header': False,
'range': 'A2:A'
}
},
unnest=True))
]
for row in get_rows(
config, task['auth_sheets'], {
'sheets': {
'sheet': task['sheet'],
'tab': 'CM Accounts',
'header': False,
'range': 'A2:A'
}
}):
if row:
account_id = lookup_id(row[0])
is_superuser, profile_id = get_profile_for_api(
config, task['auth_cm'], account_id)
kwargs = {
'profileId': profile_id,
'campaignIds': campaigns,
'archived': False
}
if is_superuser:
kwargs['accountId'] = account_id
yield from API_DCM(
config,
task['auth_cm'],
iterate=True,
internal=is_superuser).placementGroups().list(
**kwargs).execute()
def get_campaigns(accounts):
if project.verbose:
print('DCM Campaigns')
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'],
account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
for campaign in API_DCM(
'user', iterate=True,
internal=is_superuser).campaigns().list(**kwargs).execute():
if int(campaign['accountId']) in accounts:
yield [
campaign['accountId'],
campaign.get('subaccountId'),
campaign['advertiserId'],
campaign.get('advertiserGroupId'),
campaign['id'],
campaign['name'],
campaign['archived'],
campaign['startDate'],
campaign['endDate'],
campaign.get('comment', ''),
epoch_to_datetime(campaign['createInfo']['time'], 1000),
epoch_to_datetime(campaign['lastModifiedInfo']['time'],
1000),
campaign.get('externalId'),
campaign['defaultLandingPageId'],
campaign['adBlockingConfiguration']['enabled'],
campaign['adBlockingConfiguration']
['overrideClickThroughUrl'],
campaign['adBlockingConfiguration'].get('clickThroughUrl'),
campaign['adBlockingConfiguration'].get(
'creativeBundleId'),
campaign['nielsenOcrEnabled'],
]
def get_reports(accounts):
if project.verbose: print 'DCM Reports'
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'], account_id)
kwargs = { 'profileId':profile_id, 'accountId':account_id, 'scope':'ALL' } if is_superuser else { 'profileId':profile_id, 'scope':'ALL' }
for report in API_DCM("user", iterate=True, internal=is_superuser).reports().list(**kwargs).execute():
if long(report['accountId']) in accounts:
for delivery in report.get('delivery', {}).get('recipients', []):
REPORT_DELIVERIES.append((
report['ownerProfileId'],
report['accountId'],
report.get('subaccountId'),
report['id'],
report['delivery']['emailOwner'],
report['delivery'].get('emailOwnerDeliveryType'),
report['delivery'].get('message', ''),
delivery['email'],
delivery['deliveryType'],
))
yield [
report['ownerProfileId'],
report['accountId'],
report.get('subaccountId'),
report['id'],
report['name'],
report['type'],
report.get('format'),
epoch_to_datetime(report.get('lastModifiedTime'), 1000),
report.get('criteria', {}).get('startDate'),
report.get('criteria', {}).get('endDate'),
report.get('criteria', {}).get('relativeDateRange'),
report.get('schedule', {}).get('active'),
report.get('schedule', {}).get('startDate'),
report.get('schedule', {}).get('expirationDate'),
report.get('schedule', {}).get('runsOnDayOfMonth'),
report.get('schedule', {}).get('repeats'),
report.get('schedule', {}).get('every'),
','.join(report.get('schedule', {}).get('repeatsOnWeekDays', [])),
]
def get_account_name(auth, account):
""" Return the name of a DCM account given the account ID.
Args:
* auth: (string) Either user or service.
* account: (string) [account:advertiser@profile] token.
Returns:
* Profile ID.
Raises:
* If current credentials do not have a profile for this account.
"""
account_id, advertiser_ids = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
response = API_DCM(auth, internal=is_superuser).accounts().get(id=account_id, profileId=profile_id).execute()
return response["name"]
def report_list(auth, account):
""" Lists all the DCM report configurations for an account given the current credentials.
Bulletproofing: https://developers.google.com/doubleclick-advertisers/v3.2/reports/list
Args:
* auth: (string) Either user or service.
* account: (string) [account:advertiser@profile] token.
Returns:
* Iterator of JSONs.
"""
account_id, advertiser_id = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
kwargs = { 'profileId':profile_id, 'accountId':account_id } if is_superuser else { 'profileId':profile_id }
for report in API_DCM(auth, internal=is_superuser).reports().list(**kwargs).execute():
yield report
def bulkdozer_test():
print('testing bulkdozer')
if 'verify' in project.task['traffic']:
is_admin, profile_id = get_profile_for_api(
project.task['auth'], project.task['traffic']['account_id'])
for entity in project.task['traffic']['verify']:
service = getattr(API_DCM(project.task['auth'], internal=is_admin),
entity['type'])
cm_entity = service().get(profileId=profile_id,
id=entity['id']).execute()
values = entity['values']
for key in values:
if values[key] != cm_entity[key]:
raise '%s %s expected to be %s, was %s' % (
entity['type'], key, values[key], cm_entity[key])
def report_run(auth, account, report_id=None, name=None):
""" Trigger a DCM report to run by name or ID. Will do nothing if report is currently in progress.
Bulletproofing: https://developers.google.com/doubleclick-advertisers/v3.3/reports/run
Args:
* auth: (string) Either user or service.
* account: (string) [account:advertiser@profile] token.
* report_id: (int) ID of DCm report to fetch ( either or name ).
* name: (string) Name of report to fetch ( either or report_id ).
Returns:
* True if report run is executed
* False otherwise
"""
account_id, advertiser_id = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
kwargs = { 'profileId':profile_id, 'accountId':account_id } if is_superuser else { 'profileId':profile_id }
if project.verbose: print('DCM REPORT RUN INIT', report_id or name)
if report_id is None:
report = report_get(auth, account, name=name)
if report is None:
raise Exception('Report does not exist:', name)
else:
report_id = report['id']
kwargs = { 'profileId':profile_id, 'accountId':account_id } if is_superuser else { 'profileId':profile_id }
kwargs['reportId'] = report_id
files = report_files(auth, account, report_id)
latest_file_json = next(files, None)
if latest_file_json == None or latest_file_json['status'] != 'PROCESSING':
# run report if previously never run or currently not running
if project.verbose: print('RUNNING REPORT', report_id or name)
API_DCM(auth, internal=is_superuser).reports().run(**kwargs).execute()
return True
if project.verbose: print('REPORT RUN SKIPPED', report_id or name)
return False
def report_files(auth, account, report_id):
""" Lists all the files available for a given DCM report configuration.
Bulletproofing: https://developers.google.com/doubleclick-advertisers/v3.2/files/list
Args:
* auth: (string) Either user or service.
* account: (string) [account:advertiser@profile] token.
* report_id: (int) DCM report identifier.
Returns:
* Iterator of JSONs.
"""
account_id, advertiser_id = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
kwargs = { 'profileId':profile_id, 'accountId':account_id } if is_superuser else { 'profileId':profile_id }
kwargs['reportId'] = report_id
for report_file in API_DCM(auth, internal=is_superuser).reports().files().list(**kwargs).execute():
yield report_file
def load_multiple():
for row in get_rows(
config,
task['auth_sheets'],
{ 'sheets': {
'sheet': task['sheet'],
'tab': 'CM Profiles',
'header':False,
'range': 'A2:A'
}}
):
if row:
account_id = lookup_id(row[0])
is_superuser, profile_id = get_profile_for_api(config, task['auth_cm'], account_id)
kwargs = { 'profileId': profile_id, 'accountId': account_id } if is_superuser else { 'profileId': profile_id }
yield from API_DCM(
config,
task['auth_cm'],
iterate=True,
internal=is_superuser
).accounts().list(**kwargs).execute()
def get_subaccounts(accounts):
if project.verbose: print('DCM SubAccounts')
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'],
account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
for subaccount in API_DCM(
"user", iterate=True,
internal=is_superuser).subaccounts().list(**kwargs).execute():
if int(subaccount['accountId']) in accounts:
yield [
subaccount['accountId'],
subaccount['id'],
subaccount['name'],
]
def get_roles(accounts):
if project.verbose:
print('DCM Roles')
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'],
account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
for role in API_DCM(
'user', iterate=True,
internal=is_superuser).userRoles().list(**kwargs).execute():
if int(role['accountId']) in accounts:
if 'permissions' in role:
for permission in role['permissions']:
yield [
role['accountId'],
role.get('subaccountId'),
role['id'],
role['name'],
role['defaultUserRole'],
permission['name'],
permission['availability'],
]
else:
yield [
role['accountId'],
role.get('subaccountId'),
role['id'],
role['name'],
role['defaultUserRole'],
None,
None,
]
def get_advertisers(accounts):
if project.verbose: print 'DCM Advertisers'
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'], account_id)
kwargs = { 'profileId':profile_id, 'accountId':account_id } if is_superuser else { 'profileId':profile_id }
for advertiser in API_DCM("user", iterate=True, internal=is_superuser).advertisers().list(**kwargs).execute():
if long(advertiser['accountId']) in accounts:
yield [
advertiser['accountId'],
advertiser.get('subaccountId'),
advertiser['id'],
advertiser.get('advertiserGroupId'),
advertiser['name'],
advertiser['status'],
advertiser.get('defaultEmail'),
advertiser.get('clickThroughUrlSuffix'),
advertiser.get('defaultClickThroughEventTagId'),
advertiser['suspended'],
advertiser['floodlightConfigurationId'],
advertiser['originalFloodlightConfigurationId'],
]
def report_build(auth, account, body):
""" Creates a DCM report given a JSON definition.
Bulletproofing:
https://developers.google.com/doubleclick-advertisers/v3.2/reports/insert
The body JSON provided will have the following fields overriden:
* accountId - supplied as a parameter in account token.
* ownerProfileId - determined from the current credentials.
* advertiser_ids - supplied as a parameter in account token.
Args:
* auth: (string) Either user or service.
* account: (string) [account:advertiser@profile] token.
* body: (json) As defined in:
https://developers.google.com/doubleclick-advertisers/v3.2/reports#resource
Returns:
* JSON definition of report created or existing.
"""
report = report_get(auth, account, name=body['name'])
if report is None:
account_id, advertiser_ids = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
# add the account id to the body
body['accountId'] = account_id
body['ownerProfileId'] = profile_id
# add advertisers to the body, ignore for floodlight reports
if advertiser_ids and 'criteria' in body:
body['criteria']['dimensionFilters'] = body.get(
'criteria', {}).get('dimensionFilters',
[]) + [{
'kind': 'dfareporting#dimensionValue',
'dimensionName': 'dfa:advertiser',
'id': advertiser_id,
'matchType': 'EXACT'
} for advertiser_id in advertiser_ids]
# add default daily schedule if it does not exist ( convenience )
if 'schedule' not in body:
body['schedule'] = {'active': True, 'repeats': 'DAILY', 'every': 1}
# add default start and end if it does not exist ( convenience )
if 'startDate' not in body['schedule']:
body['schedule']['startDate'] = str(date.today())
# add default start and end if it does not exist ( convenience )
if 'expirationDate' not in body['schedule']:
body['schedule']['expirationDate'] = str(
(date.today() + timedelta(days=365)))
#pprint.PrettyPrinter().pprint(body)
# create the report
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
kwargs['body'] = body
report = API_DCM(
auth, internal=is_superuser).reports().insert(**kwargs).execute()
# run the report
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
kwargs['reportId'] = report['id']
API_DCM(auth, internal=is_superuser).reports().run(**kwargs).execute()
else:
if project.verbose:
print('DCM Report Exists:', body['name'])
return report
# initialize project
project.from_commandline(parser=parser)
auth = 'service' if project.args.service else 'user'
is_superuser, profile = get_profile_for_api(auth, project.args.account)
kwargs = {
'profileId': profile,
'accountId': project.args.account
} if is_superuser else {
'profileId': profile
}
# get report list
if project.args.report:
kwargs['reportId'] = project.args.report
report = API_DCM(
auth, internal=is_superuser).reports().get(**kwargs).execute()
print json.dumps(report, indent=2, sort_keys=True)
# get report files
elif project.args.files:
kwargs['reportId'] = project.args.files
for report_file in API_DCM(
auth, internal=is_superuser).reports().files().list(
**kwargs).execute():
print json.dumps(report_file, indent=2, sort_keys=True)
# get schema
elif project.args.schema:
filename, report = report_file(auth, project.args.account,
project.args.schema, None, 10)
rows = report_to_rows(report)
help='report ID to pull file list',
default=None)
parser.add_argument('--list', help='list reports', action='store_true')
# initialize project
project.from_commandline(parser=parser)
auth = 'service' if project.args.service else 'user'
kwargs = {}
if project.args.account: kwargs['accountId'] = project.args.account
if project.args.profile: kwargs['profileId'] = project.args.profile
# get report list
if project.args.report:
kwargs['reportId'] = project.args.report
report = API_DCM(auth).reports().get(**kwargs).execute()
print json.dumps(report, indent=2, sort_keys=True)
# get report files
elif project.args.files:
kwargs['reportId'] = project.args.files
for report_file in API_DCM(auth).reports().files().list(
**kwargs).execute():
print json.dumps(report_file, indent=2, sort_keys=True)
# get schema
elif project.args.schema:
filename, report = report_file(auth, project.args.account,
project.args.schema, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
def get_profiles(accounts):
if project.verbose: print('DCM Profiles')
for account_id in accounts:
is_superuser, profile_id = get_profile_for_api(project.task['auth'],
account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
for profile in API_DCM(
"user", iterate=True,
internal=is_superuser).accountUserProfiles().list(
**kwargs).execute():
if int(profile['accountId']) in accounts:
for campaign in profile.get('campaignFilter',
{}).get('objectIds', []):
PROFILE_CAMPAIGNS.append([
profile['id'],
profile['accountId'],
profile.get('subaccountId'),
campaign,
])
for site in profile.get('siteFilter', {}).get('objectIds', []):
PROFILE_SITES.append([
profile['id'],
profile['accountId'],
profile.get('subaccountId'),
site,
])
for role in profile.get('userRoleFilter',
{}).get('objectIds', []):
PROFILE_ROLES.append([
profile['id'],
profile['accountId'],
profile.get('subaccountId'),
role,
])
for advertiser in profile.get('advertiserFilter',
{}).get('objectIds', []):
PROFILE_ADVERTISERS.append([
profile['id'],
profile['accountId'],
profile.get('subaccountId'),
advertiser,
])
yield [
profile['id'],
profile['accountId'],
profile.get('subaccountId'),
profile['name'],
profile['email'],
profile.get('locale'),
profile.get('userRoleId'),
profile.get('userAccessType'),
profile['active'],
profile.get('comments', ''),
profile.get('traffickerType'),
]
def report_create(auth, account, name, config):
account_id, advertiser_ids = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
report = report_get(auth, account_id, name=name)
if report is None:
body = {
'kind': 'dfareporting#report',
'type': config.get('type', 'STANDARD').upper(),
'name': name,
'format': config.get('format', 'CSV'),
'accountId': account_id,
'delivery': {
'emailOwner': False,
'emailOwnerDeliveryType': 'LINK'
},
'schedule': {
'active': True,
'repeats': 'DAILY',
'every': 1,
'startDate': str(date.today()),
'expirationDate': str((date.today() + timedelta(days=365))),
}
}
if body['type'] == 'STANDARD': body.update(get_body_standard(config))
elif body['type'] == 'FLOODLIGHT':
body.update(get_body_floodlight(config))
if advertiser_ids:
body['criteria']['dimensionFilters'] = body['criteria'].get(
'dimensionFilters', []) + [{
'kind': 'dfareporting#dimensionValue',
'dimensionName': 'dfa:advertiser',
'id': advertiser_id,
'matchType': 'EXACT'
} for advertiser_id in advertiser_ids]
#pprint.PrettyPrinter().pprint(body)
# create the report
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
kwargs['body'] = body
report = API_DCM(
auth, internal=is_superuser).reports().insert(**kwargs).execute()
# run the report
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
kwargs['reportId'] = report['id']
API_DCM(auth,
internal=is_superuser).reports().insert(**kwargs).execute()
return report
def itp_audit_cm():
account_id = project.task['account']
# Read Advertiser Ids
advertiser_ids = list(
get_rows('service', project.task['read']['advertiser_ids']))
is_superuser, profile_id = get_profile_for_api('user', account_id)
datasets_create('user', project.id, project.task['dataset'])
placements = {}
campaignNames = {}
siteNames = {}
advertiserNames = {}
for c in API_DCM('user', iterate=True,
internal=is_superuser).campaigns().list(
archived=False,
profileId=profile_id,
accountId=account_id,
advertiserIds=advertiser_ids
if advertiser_ids else None).execute():
# only keep campaigns with end dates in the future or less than 90 days ago
if ((date.fromisoformat(c['endDate']) - date.today()).days > -90):
campaignNames[c['id']] = {
'id': c['id'],
'name': c['name'],
'startDate': c['startDate'],
'endDate': c['endDate']
}
validCampaignIds = [int(i) for i in campaignNames.keys()]
for s in API_DCM('user', iterate=True, internal=is_superuser).sites().list(
profileId=profile_id,
accountId=account_id,
campaignIds=validCampaignIds).execute():
siteNames[s['id']] = s['name']
for a in API_DCM('user', iterate=True,
internal=is_superuser).advertisers().list(
profileId=profile_id,
accountId=account_id,
ids=advertiser_ids[:500]
if advertiser_ids else None).execute():
advertiserNames[a['id']] = a['name']
if len(advertiser_ids) > 500:
for a in API_DCM('user', iterate=True,
internal=is_superuser).advertisers().list(
profileId=profile_id,
accountId=account_id,
ids=advertiser_ids[500:]
if advertiser_ids else None).execute():
advertiserNames[a['id']] = a['name']
for p in API_DCM(
'user', iterate=True, internal=is_superuser).placements().list(
archived=False,
profileId=profile_id,
accountId=account_id,
advertiserIds=advertiser_ids if advertiser_ids else None,
campaignIds=validCampaignIds).execute():
# exclude 1x1 tracking placements
if not (p['size']['height'] == 1 and p['size']['width'] == 1):
placements[p['id']] = {
'id': p['id'],
'name': p['name'],
'advertiserId': p['advertiserId'],
'advertiserName': advertiserNames[p['advertiserId']],
'campaignId': p['campaignId'],
'campaignName': campaignNames[p['campaignId']]['name'],
'siteId': p['siteId'],
'siteName': siteNames[p['siteId']],
'adsTotal': 0,
'adsNotImpacted': 0,
'adsFrequencyCapped': 0,
'adsAudienceSegmentation': 0,
'adsAudienceListTargeting': 0,
'adsDynamicCreative': 0,
'adsPlatformBrowserTargeting': 0
}
for ad in API_DCM('user', iterate=True, internal=is_superuser).ads().list(
type='AD_SERVING_STANDARD_AD',
profileId=profile_id,
accountId=account_id,
campaignIds=validCampaignIds).execute():
# only analyze standard, non-default priority ads that have been assigned 1+ placements and creatives
if ad['deliverySchedule'][
'priority'] != "AD_PRIORITY_16" and 'placementAssignments' in ad and 'creativeAssignments' in ad[
'creativeRotation']:
hasDynamicCreative = False
for creative in ad['creativeRotation']['creativeAssignments']:
if 'richMediaExitOverrides' in creative:
hasDynamicCreative = True
break
for p in ad['placementAssignments']:
if p['placementId'] in placements:
knownPlacement = placements[p['placementId']]
knownPlacement['adsTotal'] += 1
if 'frequencyCap' in ad['deliverySchedule']:
knownPlacement['adsFrequencyCapped'] += 1
if 'audienceSegmentId' in ad:
knownPlacement['adsAudienceSegmentation'] += 1
if 'remarketingListExpression' in ad:
knownPlacement['adsAudienceListTargeting'] += 1
if 'technologyTargeting' in ad and (
'browsers' in ad['technologyTargeting']
or 'platformTypes' in ad['technologyTargeting']):
knownPlacement['adsPlatformBrowserTargeting'] += 1
if hasDynamicCreative:
knownPlacement['adsDynamicCreative'] += 1
if not 'frequencyCap' in ad[
'deliverySchedule'] and not 'audienceSegmentId' in ad and not 'remarketingListExpression' in ad:
knownPlacement['adsNotImpacted'] += 1
write_data = []
for p in placements.values():
write_data.append([
p['name'] + ' - ' + str(p['id']), p['id'], p['name'],
p['advertiserName'] + ' - ' + str(p['advertiserId']),
p['advertiserId'], p['advertiserName'],
p['campaignName'] + ' - ' + str(p['campaignId']), p['campaignId'],
p['campaignName'], p['siteId'], p['siteName'], p['adsTotal'],
p['adsNotImpacted'], p['adsFrequencyCapped'],
p['adsAudienceListTargeting'], p['adsAudienceSegmentation'],
p['adsPlatformBrowserTargeting'], p['adsDynamicCreative']
])
placements_out = {}
placements_out["bigquery"] = {
"dataset": project.task['dataset'],
"table": CM_PLACEMENT_AUDIT_TABLE,
"is_incremental_load": False,
"datastudio": True,
"schema": PLACEMENTS_SCHEMA,
"skip_rows": 0
}
if placements: put_rows('service', placements_out, write_data)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to help debug CM reports and build reporting tools.
Examples:
To get list of reports: python cm.py --account [id] --list -u [user credentials path]
To get report: python cm.py --account [id] --report [id] -u [user credentials path]
To get report files: python cm.py --account [id] --files [id] -u [user credentials path]
To get report sample: python cm.py --account [id] --sample [id] -u [user credentials path]
To get report schema: python cm.py --account [id] --schema [id] -u [user credentials path]
"""))
parser.add_argument('--account',
help='Account ID to use to pull the report.',
default=None)
parser.add_argument('--report',
help='Report ID to pull JSON definition.',
default=None)
parser.add_argument('--schema',
help='Report ID to pull achema definition.',
default=None)
parser.add_argument('--sample',
help='Report ID to pull sample data.',
default=None)
parser.add_argument('--files',
help='Report ID to pull file list.',
default=None)
parser.add_argument('--list', help='List reports.', action='store_true')
# initialize project
parser = commandline_parser(parser, arguments=('-u', '-c', '-s', '-v'))
args = parser.parse_args()
config = Configuration(user=args.user,
client=args.client,
service=args.service,
verbose=args.verbose)
auth = 'service' if args.service else 'user'
is_superuser, profile = get_profile_for_api(config, auth, args.account)
kwargs = {
'profileId': profile,
'accountId': args.account
} if is_superuser else {
'profileId': profile
}
# get report list
if args.report:
kwargs['reportId'] = args.report
report = API_DCM(
config, auth,
internal=is_superuser).reports().get(**kwargs).execute()
print(json.dumps(report, indent=2, sort_keys=True))
# get report files
elif args.files:
kwargs['reportId'] = args.files
for rf in API_DCM(
config, auth, internal=is_superuser,
iterate=True).reports().files().list(**kwargs).execute():
print(json.dumps(rf, indent=2, sort_keys=True))
# get schema
elif args.schema:
filename, report = report_file(config, auth, args.account, args.schema,
None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
print(json.dumps(report_schema(next(rows)), indent=2, sort_keys=True))
# get sample
elif args.sample:
filename, report = report_file(config, auth, args.account, args.sample,
None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
for r in rows_print(rows, row_min=0, row_max=20):
pass
# get list
else:
for report in API_DCM(config,
auth,
internal=is_superuser,
iterate=True).reports().list(**kwargs).execute():
print(json.dumps(report, indent=2, sort_keys=True))
def conversions_upload(auth,
account,
floodlight_activity_id,
conversion_type,
conversion_rows,
encryption_entity=None,
update=False):
""" Uploads an offline conversion list to DCM.
BulletProofing:
https://developers.google.com/doubleclick-advertisers/guides/conversions_upload
Handles errors and segmentation of conversion so list can be any size.
Args:
* auth: (string) Either user or service.
* account: (string) [account:advertiser@profile] token.
* floodlight_activity_id: (int) ID of DCM floodlight to upload conversions
to.
* converstion_type: (string) One of the following: encryptedUserId,
encryptedUserIdCandidates, gclid, mobileDeviceId.
* conversion_rows: (iterator) List of the following rows: Ordinal,
timestampMicros, encryptedUserId | encryptedUserIdCandidates | gclid |
mobileDeviceId.
* encryption_entity: (object) See EncryptionInfo docs:
https://developers.google.com/doubleclick-advertisers/v3.2/conversions/batchinsert#encryptionInfo
"""
account_id, advertiser_id = parse_account(auth, account)
is_superuser, profile_id = get_profile_for_api(auth, account_id)
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
kwargs['id'] = floodlight_activity_id
response = API_DCM(
auth,
internal=is_superuser).floodlightActivities().get(**kwargs).execute()
# upload in batch sizes of DCM_CONVERSION_SIZE
row_count = 0
row_buffer = []
for is_last, row in flag_last(conversion_rows):
row_buffer.append(row)
if is_last or len(row_buffer) == DCM_CONVERSION_SIZE:
if project.verbose:
print('CONVERSION UPLOADING ROWS: %d - %d' %
(row_count, row_count + len(row_buffer)))
body = {
'conversions': [{
'floodlightActivityId':
floodlight_activity_id,
'floodlightConfigurationId':
response['floodlightConfigurationId'],
'ordinal':
row[0],
'timestampMicros':
row[1],
'quantity':
1,
'value':
0.0,
conversion_type:
row[2],
} for row in row_buffer]
}
if encryption_entity:
body['encryptionInfo'] = encryption_entity
kwargs = {
'profileId': profile_id,
'accountId': account_id
} if is_superuser else {
'profileId': profile_id
}
kwargs['body'] = body
if update:
results = API_DCM(
auth, internal=is_superuser).conversions().batchupdate(
**kwargs).execute()
else:
results = API_DCM(
auth, internal=is_superuser).conversions().batchinsert(
**kwargs).execute()
# stream back satus
for status in results['status']:
yield status
# clear the buffer
row_count += len(row_buffer)
row_buffer = []
def _api(self, iterate=False):
"""Returns an DCM API instance. Must be overloaded by one of the derived classes and extended for specific endpoint."""
return API_DCM(self.auth, iterate, self.is_admin)