def report_build(auth, account, body):
report = report_get(auth, account, name=body['name'])
if report is None:
account_id, advertiser_ids, profile_id = parse_account(auth, account)
service = get_service('dfareporting', API_VERSION, auth, uri_file=API_URI)
# add the account id to the body
body['accountId'] = account_id
body['ownerProfileId'] = profile_id
# add advertisers to the body
if advertiser_ids:
body['criteria']['dimensionFilters'] = body['criteria'].get('dimensionFilters', []) + [{
'kind':'dfareporting#dimensionValue',
'dimensionName':'dfa:advertiser',
'id':advertiser_id,
'matchType':'EXACT'
} for advertiser_id in advertiser_ids]
#pprint.PrettyPrinter().pprint(body)
# create the report
if INTERNAL_MODE: report = _retry(service.reports().insert(accountId=account_id, profileId=profile_id, body=body))
else: report = _retry(service.reports().insert(profileId=profile_id, body=body))
# run the report
if INTERNAL_MODE: _retry(service.reports().run(accountId=account_id, profileId=profile_id, reportId=report['id']))
else: _retry(service.reports().run(profileId=profile_id, reportId=report['id']))
else:
if project.verbose: print 'DCM Report Exists:', body['name']
return report
def object_put(auth, path, data, mimetype='application/octet-stream'):
bucket, filename = path.split(':', 1)
service = get_service('storage', 'v1', auth)
media = MediaIoBaseUpload(data,
mimetype=mimetype,
chunksize=CHUNKSIZE,
resumable=True)
request = service.objects().insert(bucket=bucket,
name=filename,
media_body=media)
response = None
errors = 0
while response is None:
error = None
try:
status, response = request.next_chunk()
if project.verbose and status:
print "Uploaded %d%%." % int(status.progress() * 100)
except HttpError, e:
if e.resp.status < 500: raise
error = e
except (httplib2.HttpLib2Error, IOError), e:
error = e
def __init__(self, auth, trix_id, feed_name, parse=True, spreadsheet=None):
"""Initializes the feed with parameters.
Args:
auth: The authentication scheme to use based on the json configuration
file.
trix_id: Unique identifier of the Google Sheet that represents the
Bulkdozer feed.
feed_name: The name of the feed to initialize.
spreadsheet: Optional, the spreadsheet object representing the Bulkdozer
feed spreadsheet, useful to limit calls to the sheets API and allow
multiple Feed objects to use the same spreadsheet instance. This is used
to determine which tabs exist in the feed so the correct one can be
selected for the entity this Feed object represents.
"""
self.auth = auth
self.trix_id = trix_id
self.trix_range = 'A1:AZ'
self.feed_name = feed_name
self._service = get_service('sheets', 'v4', auth)
self._parse = parse
# TODO: Make sure we only read the spreadsheet object or the list of tabs
# once
if spreadsheet:
self.spreadsheet = spreadsheet
else:
self.spreadsheet = self._service.spreadsheets().get(
spreadsheetId=self.trix_id).execute()
self.raw_feed = self._get_feed()
self.feed = self._feed_to_dict(parse=self._parse)
def report_build(auth, body):
report = report_get(auth, name=body['metadata']['title'])
if not report:
service = get_service('doubleclickbidmanager', API_VERSION, auth)
# add default daily schedule if it does not exist ( convenience )
if "schedule" not in body:
body['schedule'] = {
"endTimeMs": long((time.time() + (365 * 24 * 60 * 60)) * 1000), # 1 year in future
"frequency": "DAILY",
"nextRunMinuteOfDay": 4 * 60,
"nextRunTimezoneCode": body['timezoneCode']
}
#pprint.PrettyPrinter().pprint(body)
# build report
job = service.queries().createquery(body=body)
report = _retry(job)
# run report first time
body = {
"dataRange":report['metadata']['dataRange'],
"timezoneCode":report['schedule']['nextRunTimezoneCode']
}
run = service.queries().runquery(queryId=report['queryId'], body=body)
_retry(run)
else:
if project.verbose: print 'DBM Report Exists:', body['metadata']['title']
return report
def execute(self, run=True):
# start building call sequence with service object
f = get_service(self.api, self.version, self.auth, uri_file=self.uri)
# build calls along stack ( first call is different because it is a called function )
# do not call functions, as the abstract is necessary for iterator page next calls
for count, f_n in enumerate(self.function_stack):
f = getattr(f if count == 0 else f(), f_n)
# for cases where job is handled manually, save the job
job = f(**self.function_kwargs)
if run == True:
# pass arguments to the last function in the call chain
results = API_Retry(job)
# if paginated, automatically iterate
if 'nextPageToken' in results or self.iterate == True:
return API_Iterator(f, self.function_kwargs, results)
# if not pagenated, return object as is
else:
return results
# if not run, just return job object ( for chunke dupload for example )
else:
return job
def bucket_get(auth, name):
service = get_service('storage', 'v1', auth)
try:
return service.buckets().get(bucket=name).execute()
except HttpError, e:
if e.resp.status == 404: return None
elif e.resp.status in [403, 500, 503]: sleep(5)
else: raise
def __init__(self, auth, profile_id):
"""Initializes the object with a specific CM profile ID and an authorization scheme.
"""
self.service = get_service('dfareporting', self.API_VERSION, auth)
self.profile_id = profile_id
self._entity = 'UNDEFINED'
self._metrics = {}
def report_delete(auth, account, report_id = None, name=None):
account_id, advertiser_ids, profile_id = parse_account(auth, account)
report = report_get(auth, account, report_id, name)
if report:
service = get_service('dfareporting', API_VERSION, auth, uri_file=API_URI)
if INTERNAL_MODE: _retry(service.reports().delete(accountId=account_id, profileId=profile_id, reportId=report['id']))
else: _retry(service.reports().delete(profileId=profile_id, reportId=report['id']))
else:
if project.verbose: print 'DCM DELETE: No Report'
def sheets_write(auth, sheet_url, sheet_tab, sheet_range, data, valueInputOption='RAW'):
if project.verbose: print 'SHEETS WRITE', sheet_url, sheet_tab, sheet_range
service = get_service('sheets', 'v4', auth)
sheet_id = sheets_id(sheet_url)
range = sheets_tab_range(sheet_tab, sheet_range)
body = {
"values": list(data)
}
_retry(service.spreadsheets().values().update(spreadsheetId=sheet_id, range=range, body=body, valueInputOption=valueInputOption))
def create_folder(auth, parent, name):
drive = get_service('drive', 'v3', auth)
file_metadata = {
'name': name,
'parents': [parent],
'mimeType': 'application/vnd.google-apps.folder'
}
return drive.files().create(body=file_metadata, fields='id').execute()
def report_get(auth, report_id=None, name=None):
service = get_service('doubleclickbidmanager', API_VERSION, auth)
if name:
job = service.queries().listqueries()
result = _retry(job)
return ([query for query in result.get('queries', []) if query['metadata']['title'] == name ] or [None])[0]
else:
job = service.queries().getquery(queryId=report_id)
return _retry(job)
def report_delete(auth, report_id=None, name=None):
if project.verbose: print "DBM DELETE:", report_id or name
report = report_get(auth, report_id, name)
if report:
service = get_service('doubleclickbidmanager', API_VERSION, auth)
job = service.queries().deletequery(queryId=report['queryId'])
_retry(job)
else:
if project.verbose: print 'DBM DELETE: No Report'
def file_find(auth, name, parent=None):
drive = get_service('drive', 'v3', auth)
query = "trashed = false and name = '%s'" % name
if parent: query = "%s and '%s' in parents" % (query, parent)
try:
return (_retry(drive.files().list(q=query))['files'] or [None])[0]
except HttpError:
return None
def report_files(auth, account, report_id):
account_id, advertiser_id, profile_id = parse_account(auth, account)
service = get_service('dfareporting', API_VERSION, auth)
next_page = None
while next_page != '':
if INTERNAL_MODE: response = _retry(service.reports().files().list(accountId=account_id, profileId=profile_id, reportId=report_id, pageToken=next_page))
else: response = _retry(service.reports().files().list(profileId=profile_id, reportId=report_id, pageToken=next_page))
next_page = response['nextPageToken']
for report in response['items']:
yield report
def object_list(auth, path, raw=False, files_only=False):
bucket, prefix = path.split(':', 1)
service = get_service('storage', 'v1', auth)
next_page = None
while next_page != '':
response = service.objects().list(bucket=bucket,
prefix=prefix).execute()
next_page = response.get('nextPageToken', '')
for item in response.get('items', []):
if files_only and item['name'].endswith('/'): continue
yield item if raw else '%s:%s' % (bucket, item['name'])
def report_create(auth, account, name, config):
account_id, advertiser_ids, profile_id = parse_account(auth, account)
report = report_get(auth, account_id, name=name)
if report is None:
service = get_service('dfareporting', API_VERSION, auth, uri_file=API_URI)
body = {
'kind':'dfareporting#report',
'type':config.get('type', 'STANDARD').upper(),
'name':name,
'format':config.get('format', 'CSV'),
'accountId':account_id,
'delivery': {
'emailOwner':False,
'emailOwnerDeliveryType':'LINK'
},
'schedule': {
'active':True,
'repeats':'DAILY',
'every': 1,
'startDate':str(date.today()),
'expirationDate':str((date.today() + timedelta(days=365))),
}
}
if body['type'] == 'STANDARD': body.update(get_body_standard(config))
elif body['type'] == 'FLOODLIGHT': body.update(get_body_floodlight(config))
if advertiser_ids:
body['criteria']['dimensionFilters'] = body['criteria'].get('dimensionFilters', []) + [{
'kind':'dfareporting#dimensionValue',
'dimensionName':'dfa:advertiser',
'id':advertiser_id,
'matchType':'EXACT'
} for advertiser_id in advertiser_ids]
#pprint.PrettyPrinter().pprint(body)
# create the report
if INTERNAL_MODE: report = _retry(service.reports().insert(accountId=account_id, profileId=profile_id, body=body))
else: report = _retry(service.reports().insert(profileId=profile_id, body=body))
# run the report
if INTERNAL_MODE: _retry(service.reports().run(accountId=account_id, profileId=profile_id, reportId=report['id']))
else: _retry(service.reports().run(profileId=profile_id, reportId=report['id']))
return report
def lineitem_write(auth, rows, dry_run=True):
service = get_service('doubleclickbidmanager', API_VERSION, auth)
header = [s['name'] for s in LineItem_Write_Schema]
body = {
"lineItems":'%s\n%s' % (','.join(header), rows_to_csv(rows).read()), # add header row
"format":'CSV',
"dryRun":dry_run
}
job = service.lineitems().uploadlineitems(body=body)
result = _retry(job)
#print result
return result
def sheets_tab_copy(auth, from_sheet_url, from_sheet_tab, to_sheet_url, to_sheet_tab):
if project.verbose: print 'SHEETS COPY', from_sheet_url, from_sheet_tab, to_sheet_url, to_sheet_tab
service = get_service('sheets', 'v4', auth)
# convert human readable to ids
from_sheet_id, from_tab_id = sheets_tab_id(auth, from_sheet_url, from_sheet_tab)
to_sheet_id, to_tab_id = sheets_tab_id(auth, to_sheet_url, to_sheet_tab)
# overwrite only if does not exist ( PROTECTION )
if to_tab_id is None:
# copy tab between sheets ( seriously, the name changes to be "Copy of [from_sheet_tab]" )
request = _retry(service.spreadsheets().sheets().copyTo(spreadsheetId=from_sheet_id, sheetId=from_tab_id, body={
"destinationSpreadsheetId": to_sheet_id,
}))
# change the name back ( remove "Copy of " )
sheets_tab_rename(auth, to_sheet_url, 'Copy of %s' % from_sheet_tab, to_sheet_tab)
def object_copy(auth, path_from, path_to):
from_bucket, from_filename = path_from.split(':', 1)
to_bucket, to_filename = path_to.split(':', 1)
body = {
"kind": "storage#object",
"bucket": to_bucket,
"name": to_filename,
"storageClass": "REGIONAL",
}
service = get_service('storage', 'v1', auth)
return service.objects().rewrite(sourceBucket=from_bucket,
sourceObject=from_filename,
destinationBucket=to_bucket,
destinationObject=to_filename,
body=body).execute()
def get_email_attachments(auth,
email_from,
email_to,
subject_regexp=None,
attachment_regexp=None,
date_min=None,
date_max=None):
if project.verbose: print 'GETTING EMAIL ATTACHMENTS'
service = get_service('gmail', 'v1', auth)
messages = _email_find(service, email_from, email_to, date_min, date_max)
subject_filter = re.compile(r'%s' %
subject_regexp) if subject_regexp else None
for message in messages:
message = _retry(service.users().messages().get(userId='me',
id=message['id']))
if subject_filter is None or subject_filter.match(
_get_subject(message)):
yield _email_attachments(service, message, attachment_regexp)
def conversions_upload(auth, account, floodlight_activity_id, conversion_type, conversion_rows, encryption_entity=None, update=False):
account_id, advertiser_id, profile_id = parse_account(auth, account)
service = get_service('dfareporting', API_VERSION, auth)
if INTERNAL_MODE: response = _retry(service.floodlightActivities().get(accountId=account_id, profileId=profile_id, id=floodlight_activity_id))
else: response = _retry(service.floodlightActivities().get(profileId=profile_id, id=floodlight_activity_id))
# upload in batch sizes of DCM_CONVERSION_SIZE
row_count = 0
row_buffer = []
for is_last, row in flag_last(conversion_rows):
row_buffer.append(row)
if is_last or len(row_buffer) == DCM_CONVERSION_SIZE:
if project.verbose: print 'CONVERSION UPLOADING ROWS: %d - %d' % (row_count, row_count + len(row_buffer))
body = {
'conversions': [{
'floodlightActivityId': floodlight_activity_id,
'floodlightConfigurationId': response['floodlightConfigurationId'],
'ordinal': row[0],
'timestampMicros': row[1],
'quantity':1,
'value':0.0,
conversion_type: row[2],
} for row in row_buffer]
}
if encryption_entity: body['encryptionInfo'] = encryption_entity
if update:
if INTERNAL_MODE: results = _retry(service.conversions().batchupdate(accountId=account_id, profileId=profile_id, body=body))
else: results = _retry(service.conversions().batchupdate(profileId=profile_id, body=body))
else:
if INTERNAL_MODE: results = _retry(service.conversions().batchinsert(accountId=account_id, profileId=profile_id, body=body))
else: results = _retry(service.conversions().batchinsert(profileId=profile_id, body=body))
# stream back satus
for status in results['status']: yield status
# clear the buffer
row_count += len(row_buffer)
row_buffer = []
def report_fetch(auth, report_id):
service = get_service('doubleclicksearch', 'v2', auth)
if project.verbose: print 'Fetching Report', report_id
files = report_ready(service, report_id)
reports = []
i = 0
for file in files:
reports.append({
'name':
'%s_%d_%s.csv' % (report_id, i, str(date.today())),
'report_id':
report_id,
'report_fragment':
i
})
return reports
def report_file(auth, account, report_id=None, name=None, timeout=60, chunksize=DCM_CHUNK_SIZE):
account_id, advertiser_id, profile_id = parse_account(auth, account)
file_json = report_fetch(auth, account, report_id, name, timeout)
if file_json == False:
return None, None
elif file_json == True:
return 'report_running.csv', None
else:
service = get_service('dfareporting', API_VERSION, auth, uri_file=API_URI)
filename = '%s_%s.csv' % (file_json['fileName'], file_json['dateRange']['endDate'].replace('-', ''))
# streaming
if chunksize:
return filename, media_download(service.files().get_media(reportId=file_json['reportId'], fileId=file_json['id']), chunksize)
# single object
else:
return filename, StringIO(_retry(service.files().get_media(reportId=file_json['reportId'], fileId=file_json['id'])))
def bucket_create(auth, project, name):
if bucket_get(auth, name) is None:
body = {
"kind": "storage#bucket",
"name": name,
"storageClass": "REGIONAL",
"location": "us-west1",
}
service = get_service('storage', 'v1', auth)
try:
return service.buckets().insert(project=project,
body=body).execute()
sleep(1)
except HttpError, e:
if e.resp.status in [403, 500, 503]: sleep(5)
elif json.loads(e.content)['error']['code'] == 409:
pass # already exists ( ignore )
else:
raise
def init_daos():
global video_format_dao
global landing_page_dao
global campaign_dao
global creative_association_dao
global creative_dao
global placement_dao
global creative_asset_dao
global ad_dao
global event_tag_dao
global spreadsheet
service = get_service('sheets', 'v4', project.task['auth'])
spreadsheet = service.spreadsheets().get(
spreadsheetId=project.task['sheet_id']).execute()
store.auth = project.task['auth']
store.trix_id = project.task.get('store', {}).get('sheet_id',
project.task['sheet_id'])
store.load_id_map()
video_format_dao = VideoFormatDAO(project.task['auth'],
project.task['dcm_profile_id'])
landing_page_dao = LandingPageDAO(project.task['auth'],
project.task['dcm_profile_id'])
campaign_dao = CampaignDAO(project.task['auth'],
project.task['dcm_profile_id'])
creative_association_dao = CreativeAssociationDAO(
project.task['auth'], project.task['dcm_profile_id'])
creative_dao = CreativeDAO(project.task['auth'],
project.task['dcm_profile_id'])
placement_dao = PlacementDAO(project.task['auth'],
project.task['dcm_profile_id'])
creative_asset_dao = CreativeAssetDAO(project.task['auth'],
project.task['dcm_profile_id'],
project.id)
ad_dao = AdDAO(project.task['auth'], project.task['dcm_profile_id'])
event_tag_dao = EventTagDAO(project.task['auth'],
project.task['dcm_profile_id'])
def send_email(auth, email_to, email_from, email_cc, subject, text, html=None):
if project.verbose: print 'SENDING EMAIL', email_to
service = get_service('gmail', 'v1', auth)
message = MIMEMultipart('alternative')
message.set_charset('utf8')
message['to'] = email_to
message['cc'] = email_cc
message['from'] = email_from
message['subject'] = subject
text_part = MIMEText(text, 'plain', 'UTF-8')
message.attach(text_part)
if html:
html_part = MIMEText(html, 'html', 'UTF-8')
message.attach(html_part)
_retry(service.users().messages().send(
userId='me',
body={'raw': base64.urlsafe_b64encode(message.as_string())}))
def report_get(auth, account, report_id = None, name=None):
account_id, advertiser_ids, profile_id = parse_account(auth, account)
service = get_service('dfareporting', API_VERSION, auth, uri_file=API_URI)
report = None
if name:
next_page = None
while next_page != '' and report is None:
if INTERNAL_MODE: response = _retry(service.reports().list(accountId=account_id, profileId=profile_id, pageToken=next_page))
else: response = _retry(service.reports().list(profileId=profile_id, pageToken=next_page))
next_page = response['nextPageToken']
for r in response['items']:
if r['name'] == name:
report = r
break
elif report_id:
if INTERNAL_MODE: response = _retry(service.reports().get(accountId=account_id, profileId=profile_id, reportId=report_id))
else: response = _retry(service.reports().get(profileId=profile_id, reportId=report_id))
pprint.PrettyPrinter().pprint(response)
return report
def get_profile_id(auth, account_id):
service = get_service('dfareporting', API_VERSION, auth, uri_file=API_URI)
profile_admin = None
profile_network = None
for p in _retry(service.userProfiles().list())['items']:
p_id = int(p['profileId'])
a_id = int(p['accountId'])
# take the first profile for admin
if '@dcm' in p['userName']: profile_admin = p_id
elif '@dfa' in p['userName']: profile_admin = p_id
elif a_id == 2515: profile_admin = p_id
# try to find a network profile if exists
if a_id == account_id:
profile_network = p_id
break
# return admin if exists, network if exists, and finally throw exception
if profile_network or profile_admin: return profile_admin or profile_network
else: raise Exception('Add your user profile to DCM account %s.' % account_id)
def bucket_access(auth,
project,
name,
role,
emails=[],
groups=[],
services=[],
domains=[]):
service = get_service('storage', 'v1', auth)
entities = map(lambda e: 'user-%s' % e, emails) + \
map(lambda e: 'group-%s' % e, groups) + \
map(lambda e: 'user-%s' % e, services) + \
map(lambda e: 'domain-%s' % e, domains)
for entity in entities:
body = {
"kind": "storage#bucketAccessControl",
"bucket": name,
"entity": entity,
"role": role
}
_retry(service.bucketAccessControls().insert(bucket=name, body=body))
def lineitem_read(auth, advertisers=[], insertion_orders=[], lineitems=[]):
service = get_service('doubleclickbidmanager', API_VERSION, auth)
body = {
'format':'CSV',
'fileSpec':'EWF'
}
if advertisers:
body['filterType'] = 'ADVERTISER_ID'
body['filterIds'] = list(advertisers) # in case its a generator
elif insertion_orders:
body['filterType'] = 'INSERTION_ORDER_ID'
body['filterIds'] = list(insertion_orders) # in case its a generator
elif lineitems:
body['filterType'] = 'LINE_ITEM_ID'
body['filterIds'] = list(lineitems) # in case its a generator
#print body
result = _retry(service.lineitems().downloadlineitems(body=body))
for count, row in enumerate(csv_to_rows(result.get('lineItems', ''))):
if count == 0: continue # skip header
row[0] = int(row[0] or 0) # LineItem ID
row[2] = int(row[2] or 0) # Partner ID
row[11] = float(row[11] or 0) # IO Budget Amount
row[18] = float(row[18] or 0) # LineItem Budget Amount
row[21] = float(row[21] or 0) # LineItem Pacing Amount
row[23] = int(row[23] or 0) # LineItem Frequency Exposures
row[25] = int(row[25] or 0) # LineItem Frequency Amount
row[26] = float(row[26] or 0) # Bid Price (CPM)
row[28] = float(row[28] or 0) # Partner Revenue Amount
yield row
