def google_api_execute(auth, api_call, results, errors, limit=None):
try:
rows = API(api_call).execute()
if results:
# check if single object needs conversion to rows
if isinstance(rows, dict):
rows = [rows]
rows = map(lambda r: Discovery_To_BigQuery.clean(r), rows)
put_rows(auth, results, rows)
if 'bigquery' in results:
results['bigquery']['disposition'] = 'WRITE_APPEND'
except HttpError as e:
if errors:
rows = [{
'Error':
str(e),
'Parameters': [{
'Key': k,
'Value': str(v)
} for k, v in api_call['kwargs'].items()]
}]
put_rows(auth, errors, rows)
if 'bigquery' in errors:
errors['bigquery']['disposition'] = 'WRITE_APPEND'
else:
raise e
def google_api():
if project.verbose:
print('GOOGLE_API', project.task['api'], project.task['version'])
results = API(project.task).execute()
put_rows(project.task['auth'], project.task['out'], results)
def google_api():
if project.verbose:
print 'GOOGLE_API', project.task['api'], project.task['version']
results = API(project.task).execute()
put_rows(
project.task['auth'], project.task['out'], '%s_%s.json' %
(project.task['function'].replace('.', '_'), project.date), results)
def google_api_execute(config, auth, api_call, results, errors, limit=None):
"""Execute the actual API call and write to the end points defined.
The API call is completely defined at this point.
The results and error definition is optional.
Args:
auth (string): either "user" or "service" to make the API call.
api_call (dict): the JSON for the API call as defined in recipe.
results (dict): defines where the data will be written
errors (dict): defines where the errors will be written
limit (int): Reduce the number of calls ( mostly for debugging )
Returns (dict):
None, all data is transfered between API / BigQuery
Raises:
ValueError: If a required key in the recipe is missing.
"""
try:
rows = API(config, api_call).execute()
if results:
# check if single object needs conversion to rows
if isinstance(rows, dict):
rows = [rows]
# check if simple string API results
elif results.get('bigquery', {}).get('format', 'JSON') == 'CSV':
rows = [[r] for r in rows]
rows = map(lambda r: Discovery_To_BigQuery.clean(r), rows)
put_rows(config, auth, results, rows)
if 'bigquery' in results:
results['bigquery']['disposition'] = 'WRITE_APPEND'
except HttpError as e:
if errors:
rows = [{
'Error':
str(e),
'Parameters': [{
'Key': k,
'Value': str(v)
} for k, v in api_call['kwargs'].items()]
}]
put_rows(config, auth, errors, rows)
if 'bigquery' in errors:
errors['bigquery']['disposition'] = 'WRITE_APPEND'
else:
raise e
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line interface for running Google API calls. Any API works. Allows developers to quickly test
and debug API calls before building them into scripts. Useful for debugging permission or call errors.
Examples:
- Pull a DBM report via API.
- https://developers.google.com/bid-manager/v1/queries/getquery
- python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]
- Pull a list of placements:
- https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
- python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]
"""))
# get parameters
parser.add_argument('-api', help='api to run, name of product api')
parser.add_argument('-version', help='version of api')
parser.add_argument('-function', help='function to call in api')
parser.add_argument('-uri', help='function to call in api', default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate',
help='set to true to force iteration',
action='store_true')
# initialize project ( used to load standard credentials parameters )
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))
# the api wrapper takes parameters as JSON
job = {
'auth': 'service' if project.args.service else 'user',
'api': project.args.api,
'version': project.args.version,
'function': project.args.function,
'uri': project.args.uri,
'kwargs': json.loads(project.args.kwargs),
'iterate': project.args.iterate,
}
# run the API call
results = API(job).execute()
# display results
if project.args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line interface for running Google API calls. Any API works. Allows developers to quickly test
and debug API calls before building them into scripts. Useful for debugging permission or call errors.
Examples:
- Pull a DBM report via API.
- https://developers.google.com/bid-manager/v1/queries/getquery
- python google_api.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]
- Pull a list of placements:
- https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
- python google_api.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]
- Show schema for Campaign Manager advertiser list endpoint.
- https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list
- python google_api.py -api dfareporting -version v3.4 -function advertisers.list --schema
- python google_api.py -api dfareporting -version v3.4 -function Advertiser --object
- python google_api.py -api dfareporting -version v3.4 -function Advertiser --struct
"""))
# get parameters
parser.add_argument('-api', help='api to run, name of product api')
parser.add_argument('-version', help='version of api')
parser.add_argument('-function',
help='function or resource to call in api')
parser.add_argument('-uri', help='uri to use in api', default=None)
parser.add_argument('-developer-token',
help='developer token to pass in header',
default=None)
parser.add_argument(
'-login-customer-id',
help='customer to log in with when manipulating an MCC',
default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate',
help='force iteration',
action='store_true')
parser.add_argument('--limit',
type=int,
help='optional, number of records to return',
default=None)
parser.add_argument(
'--schema',
help='return function as BigQuery schema, function = [endpoint.method]',
action='store_true')
parser.add_argument(
'--object',
help=
'return resource as JSON discovery document, function = [resource]',
action='store_true')
parser.add_argument(
'--struct',
help='return resource as BigQuery structure, function = [resource]',
action='store_true')
# initialize project
parser = commandline_parser(parser,
arguments=('-u', '-c', '-s', '-k', '-v'))
args = parser.parse_args()
config = Configuration(user=args.user,
client=args.client,
service=args.service,
key=args.key,
verbose=args.verbose)
# show schema
if args.object:
print(
json.dumps(Discovery_To_BigQuery(
args.api, args.version).resource_json(args.function),
indent=2,
default=str))
elif args.struct:
print(
Discovery_To_BigQuery(args.api,
args.version).resource_struct(args.function))
# show schema
elif args.schema:
print(
json.dumps(Discovery_To_BigQuery(
args.api, args.version).method_schema(args.function),
indent=2,
default=str))
# or fetch results
else:
# the api wrapper takes parameters as JSON
job = {
'auth': 'service' if args.service else 'user',
'api': args.api,
'version': args.version,
'function': args.function,
'key': args.key,
'uri': args.uri,
'kwargs': json.loads(args.kwargs),
'headers': {},
'iterate': args.iterate,
'limit': args.limit,
}
if args.developer_token:
job['headers']['developer-token'] = args.developer_token
if args.login_customer_id:
job['headers']['login-customer-id'] = args.login_customer_id
# run the API call
results = API(job).execute()
# display results
if args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)
parser.add_argument('-function', help='function to call in api')
parser.add_argument('-uri', help='function to call in api', default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate',
help='set to true to force iteration',
action='store_true')
# initialize project ( used to load standard credentials parameters )
project.from_commandline(parser=parser)
# the api wrapper takes parameters as JSON
job = {
"auth": 'service' if project.args.service else 'user',
"api": project.args.api,
"version": project.args.version,
"function": project.args.function,
"uri": project.args.uri,
"kwargs": json.loads(project.args.kwargs),
"iterate": project.args.iterate,
}
results = API(job).execute()
if project.args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line interface for running Google API calls. Any API works. Allows developers to quickly test
and debug API calls before building them into scripts. Useful for debugging permission or call errors.
Examples:
- Pull a DBM report via API.
- https://developers.google.com/bid-manager/v1/queries/getquery
- python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]
- Pull a list of placements:
- https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
- python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]
- Show schema for Campaign Manager advertiser list endpoint.
- https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list
- python starthinker/task/google_api/helper.py -api dfareporting -version v3.4 -function advertisers.list --schema
"""))
# get parameters
parser.add_argument('-api', help='api to run, name of product api')
parser.add_argument('-version', help='version of api')
parser.add_argument('-function', help='function to call in api')
parser.add_argument('-uri', help='uri to use in api', default=None)
parser.add_argument(
'-developer-token',
help='developer token to pass in header',
default=None)
parser.add_argument(
'-login-customer-id',
help='customer to log in with when manipulating an MCC',
default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate', help='force iteration', action='store_true')
parser.add_argument(
'--schema',
help='return schema instead, function = [endpoint.method]',
action='store_true')
# initialize project ( used to load standard credentials parameters )
project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-k', '-v'))
# show schema
if project.args.schema:
endpoint, method = project.args.function.rsplit('.', 1)
print(json.dumps(Discovery_To_BigQuery(project.args.api, project.args.version).method_schema(endpoint, method), indent=2, default=str))
# or fetch results
else:
# the api wrapper takes parameters as JSON
job = {
'auth': 'service' if project.args.service else 'user',
'api': project.args.api,
'version': project.args.version,
'function': project.args.function,
'key': project.args.key,
'uri': project.args.uri,
'kwargs': json.loads(project.args.kwargs),
'headers': {},
'iterate': project.args.iterate,
}
if project.args.developer_token:
job['headers']['developer-token'] = project.args.developer_token
if project.args.login_customer_id:
job['headers']['login-customer-id'] = project.args.login_customer_id
# run the API call
results = API(job).execute()
# display results
if project.args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)