def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line utility to download and print line items from the DV360 Beta API.
Example: python helper.py --advertiser 3721733 --user user.json
python helper.py --advertiser 3721733 --service service.json
"""))
# lineitem list requires an advertiser id
parser.add_argument('--advertiser',
help='Advertiser ID to pull line items from.')
# initialize project
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))
# determine auth based on parameters
auth = 'service' if project.args.service else 'user'
# pull the line items
lineitems = API_DV360_Beta(
auth, iterate=True).advertisers().lineItems().list(
advertiserId=project.args.advertiser).execute()
# print line items
for lineitem in lineitems:
print(json.dumps(lineitem, indent=2, sort_keys=True))
def main():
# get parameters
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to get table schema from BigQuery.
This is a helper to help developers debug and create tables.
Example: `python helper.py --project [id] --dataset [name] --table [name] -s [credentials]`
"""))
parser.add_argument(
'--dataset', '-d', help='name of BigQuery dataset', default=None)
parser.add_argument(
'--table', '-t', help='name of BigQuery table', default=None)
# initialize project
project.from_commandline(
parser=parser, arguments=('-u', '-c', '-s', '-v', '-p'))
auth = 'service' if project.args.service else 'user'
# print schema
print(
json.dumps(
table_to_schema(auth, project.id, project.args.dataset,
project.args.table)['fields'],
indent=2))
def main():
# initialize project
project.from_commandline()
# run tasks and return exit code
sys.exit(project.execute())
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to transform excel sheets into csv files.
Prints to STDOUT, user is expected to pipe output into file.
Typically used for BigQuery data imports.
Examples:
List sheets in workbook: python helper.py [EXCEL FILE] --list
Convert excel to CSV: python helper.py [EXCEL FILE] --sheet [SHEET NAME] > results.csv
"""))
parser.add_argument('workbook', help='name of file to pull the rows.')
parser.add_argument('--sheet',
help='Sheet to pull the rows.',
default=None)
parser.add_argument('--list', help='List reports.', action='store_true')
# initialize project ( not really used but consistent with all helpers )
project.from_commandline(parser=parser, arguments=('-v', ))
with open(project.args.workbook, 'rb') as excel_file:
if project.args.list:
for sheet in excel_to_sheets(excel_file):
print(sheet)
elif project.args.sheet:
for sheet, row in excel_to_rows(excel_file, project.args.sheet):
print(rows_to_csv(row).read())
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line interface for running Google API calls. Any API works. Allows developers to quickly test
and debug API calls before building them into scripts. Useful for debugging permission or call errors.
Examples:
- Pull a DBM report via API.
- https://developers.google.com/bid-manager/v1/queries/getquery
- python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]
- Pull a list of placements:
- https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
- python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]
"""))
# get parameters
parser.add_argument('-api', help='api to run, name of product api')
parser.add_argument('-version', help='version of api')
parser.add_argument('-function', help='function to call in api')
parser.add_argument('-uri', help='function to call in api', default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate',
help='set to true to force iteration',
action='store_true')
# initialize project ( used to load standard credentials parameters )
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))
# the api wrapper takes parameters as JSON
job = {
'auth': 'service' if project.args.service else 'user',
'api': project.args.api,
'version': project.args.version,
'function': project.args.function,
'uri': project.args.uri,
'kwargs': json.loads(project.args.kwargs),
'iterate': project.args.iterate,
}
# run the API call
results = API(job).execute()
# display results
if project.args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
Command line to help debug DV360 reports and build reporting tools.
Examples:
To get list of reports: python helper.py --list -u [user credentials path]
To get report json: python helper.py --report [id] -u [user credentials path]
To get report schema: python helper.py --schema [id] -u [user credentials path]
To get report sample: python helper.py --sample [id] -u [user credentials path]
'''))
# create parameters
parser.add_argument('--report', help='report ID to pull json definition', default=None)
parser.add_argument('--schema', help='report ID to pull schema format', default=None)
parser.add_argument('--sample', help='report ID to pull sample data', default=None)
parser.add_argument('--list', help='list reports', action='store_true')
# initialize project
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))
auth = 'service' if project.args.service else 'user'
# get report
if project.args.report:
report = API_DBM(auth).queries().getquery(queryId=project.args.report).execute()
print(json.dumps(report, indent=2, sort_keys=True))
# get schema
elif project.args.schema:
filename, report = report_file(auth, project.args.schema, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
print(json.dumps(get_schema(rows)[1], indent=2, sort_keys=True))
# get sample
elif project.args.sample:
filename, report = report_file(auth, project.args.sample, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
for r in rows_print(rows, row_min=0, row_max=20): pass
# get list
else:
for report in API_DBM(auth, iterate=True).queries().listqueries().execute():
print(json.dumps(report, indent=2, sort_keys=True))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
Creates USER credentials from Google Cloud Project CLIENT Credentials and displays profile information if it worked.
CLIENT credentials are required to run this script, to obtain the JSON file...
Step 1: Configure Authentication Consent Screen ( do only once )
----------------------------------------
A. Visit: https://console.developers.google.com/apis/credentials/consent
B. Choose Internal if you have GSuite, otherwise choose External.
C. For Application Name enter: StarThinker
D. All other fields are optional, click Save.
Step 2: Create CLIENT Credentials ( do only once )
----------------------------------------
A. Visit: https://console.developers.google.com/apis/credentials/oauthclient
B. Choose Desktop.
C. For Name enter: StarThinker.
D. Click Create and ignore the confirmation pop-up.
Step 3: Download CLIENT Credentials File ( do only once )"
----------------------------------------"
A. Visit: https://console.developers.google.com/apis/credentials"
B. Find your newly created key under OAuth 2.0 Client IDs and click download arrow on the right."
C. The downloaded file is the CLIENT credentials, use its path for the --client -c parameter.
Step 4: Generate USER Credentials File ( do only once )"
----------------------------------------"
A. Run this command with parameters -c [CLIENT file path] and -u [USER file path].
B. The USER file will be created and can be used to access Google APIs.
C. The user profile will be printed to the screen
Example: python helper.py -u [CLIENT file path] -c [USER file path]
'''))
# initialize project, enable only some default arguments
project.from_commandline(parser=parser, arguments=('-c', '-u'))
# get profile to verify everything worked
print('Profile:', json.dumps(get_profile(), indent=2, sort_keys=True))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
Command line interface for fetching line items via legacy API.
Helps developers quickly debug lineitem issues or permissions access issues.
Example: python helper.py [line item id] -u [user credentials]
'''))
parser = argparse.ArgumentParser()
parser.add_argument('lineitem', help='lineitem ID to pull schema, or "list" to get index')
# initialize project
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))
auth = 'service' if project.args.service else 'user'
# print lineitem
for row in lineitem_read(auth, lineitems=[project.args.lineitem]):
print(row)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent('''\
Command line to send email template via gMail.
Email templates are JSON that assembles into both HTMl and TXT parts of an email.
For email sample see: https://github.com/google/starthinker/blob/master/starthinker/task/newsletter/sample.json
Example:
- Generate an HTML page from a template, then view via browser.
python helper.py --template sample.json > ~/Downloads/email.html
- Send an email template via gMail.
python helper.py --template sample.json --email_to [email protected] --email_from [email protected] -u $STARTHINKER_USER
'''))
# get parameters
parser.add_argument('--template', help='template to use for email', default=None, required=True)
parser.add_argument('--email_to', help='email to', default=None)
parser.add_argument('--email_from', help='email from', default=None)
# initialize project
project.from_commandline(parser=parser, arguments=('-u', '-c', '-v'))
# load template
with open(project.args.template, 'r') as json_file:
email = EmailTemplate(json.load(json_file))
# send or print
if project.args.email_to and project.args.email_from:
print('EMAILING: ', project.args.email_to)
send_email('user', project.args.email_to, project.args.email_from, None, email.get_subject(), email.get_text(), email.get_html())
else:
# write to STDOUT
print(email.get_html())
print('<pre style="width:600px;margin:0px auto;">%s</pre>' % email.get_text())
from starthinker.util.google_api import API_DBM
from starthinker.util.dbm import report_file, report_to_rows, report_clean
from starthinker.util.bigquery import get_schema
from starthinker.util.csv import rows_to_type, rows_print
if __name__ == "__main__":
# get parameters
parser = argparse.ArgumentParser()
parser.add_argument('--report', help='report ID to pull json definition', default=None)
parser.add_argument('--schema', help='report ID to pull schema format', default=None)
parser.add_argument('--sample', help='report ID to pull sample data', default=None)
parser.add_argument('--list', help='list reports', action='store_true')
# initialize project
project.from_commandline(parser=parser)
auth = 'service' if project.args.service else 'user'
# get report
if project.args.report:
report = API_DBM(auth).queries().getquery(queryId=project.args.report).execute()
print json.dumps(report, indent=2, sort_keys=True)
# get schema
elif project.args.schema:
filename, report = report_file(auth, project.args.schema, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
print json.dumps(get_schema(rows)[1], indent=2, sort_keys=True)
###########################################################################
#
# Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
###########################################################################
from starthinker.util.project import project
from starthinker.util.salesforce import get_service
if __name__ == '__main__':
project.from_commandline('setup')
service = get_service()
print('Credentials Ready: %s' % project.recipe['setup']['auth']['salesforce'])
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to help debug CM reports and build reporting tools.
Examples:
To get list of reports: python helper.py --account [id] --list -u [user credentials path]
To get report: python helper.py --account [id] --report [id] -u [user credentials path]
To get report files: python helper.py --account [id] --files [id] -u [user credentials path]
To get report sample: python helper.py --account [id] --sample [id] -u [user credentials path]
To get report schema: python helper.py --account [id] --schema [id] -u [user credentials path]
"""))
parser.add_argument(
'--account', help='Account ID to use to pull the report.', default=None)
parser.add_argument(
'--report', help='Report ID to pull JSON definition.', default=None)
parser.add_argument(
'--schema', help='Report ID to pull achema definition.', default=None)
parser.add_argument(
'--sample', help='Report ID to pull sample data.', default=None)
parser.add_argument(
'--files', help='Report ID to pull file list.', default=None)
parser.add_argument('--list', help='List reports.', action='store_true')
# initialize project
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))
auth = 'service' if project.args.service else 'user'
is_superuser, profile = get_profile_for_api(auth, project.args.account)
kwargs = {
'profileId': profile,
'accountId': project.args.account
} if is_superuser else {
'profileId': profile
}
# get report list
if project.args.report:
kwargs['reportId'] = project.args.report
report = API_DCM(
auth, internal=is_superuser).reports().get(**kwargs).execute()
print(json.dumps(report, indent=2, sort_keys=True))
# get report files
elif project.args.files:
kwargs['reportId'] = project.args.files
for rf in API_DCM(
auth, internal=is_superuser).reports().files().list(**kwargs).execute():
print(json.dumps(rf, indent=2, sort_keys=True))
# get schema
elif project.args.schema:
filename, report = report_file(auth, project.args.account,
project.args.schema, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
print(json.dumps(report_schema(next(rows)), indent=2, sort_keys=True))
# get sample
elif project.args.sample:
filename, report = report_file(auth, project.args.account,
project.args.sample, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
for r in rows_print(rows, row_min=0, row_max=20):
pass
# get list
else:
for report in API_DCM(
auth, internal=is_superuser).reports().list(**kwargs).execute():
print(json.dumps(report, indent=2, sort_keys=True))
def main():
# get parameters
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to get table schema from BigQuery.
Helps developers upload data to BigQuery and pull schemas. These are the
most common BigQuery tasks when developing solutions.
Examples:
Display table schema: `python helper.py --project [id] --dataset [name] --table [name] -s [credentials]`
Upload csv table: `python helper.py --project [id] --dataset [name] --table [name] --csv [file] --schema [file] -s [credentials]`
Upload excel sheet: `python helper.py --project [id] --dataset [name] --table [name] --excel_file [file] --excel_sheet [name] --schema [file] -s [credentials]`
"""))
parser.add_argument('--dataset',
help='name of BigQuery dataset',
default=None)
parser.add_argument('--table', help='name of BigQuery table', default=None)
parser.add_argument('--csv', help='CSV file path', default=None)
parser.add_argument('--schema', help='SCHEMA file path', default=None)
parser.add_argument('--excel_workbook',
help='Excel file path',
default=None)
parser.add_argument('--excel_sheet', help='Excel sheet name', default=None)
# initialize project
project.from_commandline(parser=parser,
arguments=('-u', '-c', '-s', '-v', '-p'))
auth = 'service' if project.args.service else 'user'
schema = json.loads(project.args.schema) if project.args.schema else None
if project.args.csv:
with open(project.args.csv, 'r') as csv_file:
rows = csv_to_rows(csv_file.read())
if not schema:
rows, schema = get_schema(rows)
print('DETECETED SCHEMA', json.dumps(schema))
print('Please run again with the above schema provided.')
exit()
rows_to_table(auth, project.id, project.args.dataset,
project.args.table, rows, schema)
elif project.args.excel_workbook and project.args.excel_sheet:
with open(project.args.excel_workbook, 'r') as excel_file:
rows = excel_to_rows(excel_file, project.args.excel_sheet)
if not schema:
rows, schema = get_schema(rows)
print('DETECETED SCHEMA', json.dumps(schema))
print('Please run again with the above schema provided.')
exit()
rows_to_table(auth, project.id, project.args.dataset,
project.args.table, rows, schema)
else:
# print schema
print(
json.dumps(table_to_schema(auth, project.id, project.args.dataset,
project.args.table),
indent=2))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line interface for running Google API calls. Any API works. Allows developers to quickly test
and debug API calls before building them into scripts. Useful for debugging permission or call errors.
Examples:
- Pull a DBM report via API.
- https://developers.google.com/bid-manager/v1/queries/getquery
- python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]
- Pull a list of placements:
- https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
- python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]
- Show schema for Campaign Manager advertiser list endpoint.
- https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list
- python starthinker/task/google_api/helper.py -api dfareporting -version v3.4 -function advertisers.list --schema
"""))
# get parameters
parser.add_argument('-api', help='api to run, name of product api')
parser.add_argument('-version', help='version of api')
parser.add_argument('-function', help='function to call in api')
parser.add_argument('-uri', help='uri to use in api', default=None)
parser.add_argument(
'-developer-token',
help='developer token to pass in header',
default=None)
parser.add_argument(
'-login-customer-id',
help='customer to log in with when manipulating an MCC',
default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate', help='force iteration', action='store_true')
parser.add_argument(
'--schema',
help='return schema instead, function = [endpoint.method]',
action='store_true')
# initialize project ( used to load standard credentials parameters )
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-k', '-v'))
# show schema
if project.args.schema:
endpoint, method = project.args.function.rsplit('.', 1)
print(json.dumps(Discovery_To_BigQuery(project.args.api, project.args.version).method_schema(endpoint, method), indent=2, default=str))
# or fetch results
else:
# the api wrapper takes parameters as JSON
job = {
'auth': 'service' if project.args.service else 'user',
'api': project.args.api,
'version': project.args.version,
'function': project.args.function,
'key': project.args.key,
'uri': project.args.uri,
'kwargs': json.loads(project.args.kwargs),
'headers': {},
'iterate': project.args.iterate,
}
if project.args.developer_token:
job['headers']['developer-token'] = project.args.developer_token
if project.args.login_customer_id:
job['headers']['login-customer-id'] = project.args.login_customer_id
# run the API call
results = API(job).execute()
# display results
if project.args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)
