def google_api_execute(auth, api_call, results, errors, limit=None): try: rows = API(api_call).execute() if results: # check if single object needs conversion to rows if isinstance(rows, dict): rows = [rows] rows = map(lambda r: Discovery_To_BigQuery.clean(r), rows) put_rows(auth, results, rows) if 'bigquery' in results: results['bigquery']['disposition'] = 'WRITE_APPEND' except HttpError as e: if errors: rows = [{ 'Error': str(e), 'Parameters': [{ 'Key': k, 'Value': str(v) } for k, v in api_call['kwargs'].items()] }] put_rows(auth, errors, rows) if 'bigquery' in errors: errors['bigquery']['disposition'] = 'WRITE_APPEND' else: raise e
def google_api(): if project.verbose: print('GOOGLE_API', project.task['api'], project.task['version']) results = API(project.task).execute() put_rows(project.task['auth'], project.task['out'], results)
def google_api(): if project.verbose: print 'GOOGLE_API', project.task['api'], project.task['version'] results = API(project.task).execute() put_rows( project.task['auth'], project.task['out'], '%s_%s.json' % (project.task['function'].replace('.', '_'), project.date), results)
def google_api_execute(config, auth, api_call, results, errors, limit=None): """Execute the actual API call and write to the end points defined. The API call is completely defined at this point. The results and error definition is optional. Args: auth (string): either "user" or "service" to make the API call. api_call (dict): the JSON for the API call as defined in recipe. results (dict): defines where the data will be written errors (dict): defines where the errors will be written limit (int): Reduce the number of calls ( mostly for debugging ) Returns (dict): None, all data is transfered between API / BigQuery Raises: ValueError: If a required key in the recipe is missing. """ try: rows = API(config, api_call).execute() if results: # check if single object needs conversion to rows if isinstance(rows, dict): rows = [rows] # check if simple string API results elif results.get('bigquery', {}).get('format', 'JSON') == 'CSV': rows = [[r] for r in rows] rows = map(lambda r: Discovery_To_BigQuery.clean(r), rows) put_rows(config, auth, results, rows) if 'bigquery' in results: results['bigquery']['disposition'] = 'WRITE_APPEND' except HttpError as e: if errors: rows = [{ 'Error': str(e), 'Parameters': [{ 'Key': k, 'Value': str(v) } for k, v in api_call['kwargs'].items()] }] put_rows(config, auth, errors, rows) if 'bigquery' in errors: errors['bigquery']['disposition'] = 'WRITE_APPEND' else: raise e
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent("""\ Command line interface for running Google API calls. Any API works. Allows developers to quickly test and debug API calls before building them into scripts. Useful for debugging permission or call errors. Examples: - Pull a DBM report via API. - https://developers.google.com/bid-manager/v1/queries/getquery - python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path] - Pull a list of placements: - https://developers.google.com/doubleclick-advertisers/v3.3/placements/list - python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path] """)) # get parameters parser.add_argument('-api', help='api to run, name of product api') parser.add_argument('-version', help='version of api') parser.add_argument('-function', help='function to call in api') parser.add_argument('-uri', help='function to call in api', default=None) parser.add_argument( '-kwargs', help='kwargs to pass to function, json string of name:value pairs') parser.add_argument('--iterate', help='set to true to force iteration', action='store_true') # initialize project ( used to load standard credentials parameters ) project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v')) # the api wrapper takes parameters as JSON job = { 'auth': 'service' if project.args.service else 'user', 'api': project.args.api, 'version': project.args.version, 'function': project.args.function, 'uri': project.args.uri, 'kwargs': json.loads(project.args.kwargs), 'iterate': project.args.iterate, } # run the API call results = API(job).execute() # display results if project.args.iterate: for result in results: pprint.PrettyPrinter().pprint(result) else: pprint.PrettyPrinter().pprint(results)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent("""\ Command line interface for running Google API calls. Any API works. Allows developers to quickly test and debug API calls before building them into scripts. Useful for debugging permission or call errors. Examples: - Pull a DBM report via API. - https://developers.google.com/bid-manager/v1/queries/getquery - python google_api.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path] - Pull a list of placements: - https://developers.google.com/doubleclick-advertisers/v3.3/placements/list - python google_api.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path] - Show schema for Campaign Manager advertiser list endpoint. - https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list - python google_api.py -api dfareporting -version v3.4 -function advertisers.list --schema - python google_api.py -api dfareporting -version v3.4 -function Advertiser --object - python google_api.py -api dfareporting -version v3.4 -function Advertiser --struct """)) # get parameters parser.add_argument('-api', help='api to run, name of product api') parser.add_argument('-version', help='version of api') parser.add_argument('-function', help='function or resource to call in api') parser.add_argument('-uri', help='uri to use in api', default=None) parser.add_argument('-developer-token', help='developer token to pass in header', default=None) parser.add_argument( '-login-customer-id', help='customer to log in with when manipulating an MCC', default=None) parser.add_argument( '-kwargs', help='kwargs to pass to function, json string of name:value pairs') parser.add_argument('--iterate', help='force iteration', action='store_true') parser.add_argument('--limit', type=int, help='optional, number of records to return', default=None) parser.add_argument( '--schema', help='return function as BigQuery schema, function = [endpoint.method]', action='store_true') parser.add_argument( '--object', help= 'return resource as JSON discovery document, function = [resource]', action='store_true') parser.add_argument( '--struct', help='return resource as BigQuery structure, function = [resource]', action='store_true') # initialize project parser = commandline_parser(parser, arguments=('-u', '-c', '-s', '-k', '-v')) args = parser.parse_args() config = Configuration(user=args.user, client=args.client, service=args.service, key=args.key, verbose=args.verbose) # show schema if args.object: print( json.dumps(Discovery_To_BigQuery( args.api, args.version).resource_json(args.function), indent=2, default=str)) elif args.struct: print( Discovery_To_BigQuery(args.api, args.version).resource_struct(args.function)) # show schema elif args.schema: print( json.dumps(Discovery_To_BigQuery( args.api, args.version).method_schema(args.function), indent=2, default=str)) # or fetch results else: # the api wrapper takes parameters as JSON job = { 'auth': 'service' if args.service else 'user', 'api': args.api, 'version': args.version, 'function': args.function, 'key': args.key, 'uri': args.uri, 'kwargs': json.loads(args.kwargs), 'headers': {}, 'iterate': args.iterate, 'limit': args.limit, } if args.developer_token: job['headers']['developer-token'] = args.developer_token if args.login_customer_id: job['headers']['login-customer-id'] = args.login_customer_id # run the API call results = API(job).execute() # display results if args.iterate: for result in results: pprint.PrettyPrinter().pprint(result) else: pprint.PrettyPrinter().pprint(results)
parser.add_argument('-function', help='function to call in api') parser.add_argument('-uri', help='function to call in api', default=None) parser.add_argument( '-kwargs', help='kwargs to pass to function, json string of name:value pairs') parser.add_argument('--iterate', help='set to true to force iteration', action='store_true') # initialize project ( used to load standard credentials parameters ) project.from_commandline(parser=parser) # the api wrapper takes parameters as JSON job = { "auth": 'service' if project.args.service else 'user', "api": project.args.api, "version": project.args.version, "function": project.args.function, "uri": project.args.uri, "kwargs": json.loads(project.args.kwargs), "iterate": project.args.iterate, } results = API(job).execute() if project.args.iterate: for result in results: pprint.PrettyPrinter().pprint(result) else: pprint.PrettyPrinter().pprint(results)
def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent("""\ Command line interface for running Google API calls. Any API works. Allows developers to quickly test and debug API calls before building them into scripts. Useful for debugging permission or call errors. Examples: - Pull a DBM report via API. - https://developers.google.com/bid-manager/v1/queries/getquery - python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path] - Pull a list of placements: - https://developers.google.com/doubleclick-advertisers/v3.3/placements/list - python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path] - Show schema for Campaign Manager advertiser list endpoint. - https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list - python starthinker/task/google_api/helper.py -api dfareporting -version v3.4 -function advertisers.list --schema """)) # get parameters parser.add_argument('-api', help='api to run, name of product api') parser.add_argument('-version', help='version of api') parser.add_argument('-function', help='function to call in api') parser.add_argument('-uri', help='uri to use in api', default=None) parser.add_argument( '-developer-token', help='developer token to pass in header', default=None) parser.add_argument( '-login-customer-id', help='customer to log in with when manipulating an MCC', default=None) parser.add_argument( '-kwargs', help='kwargs to pass to function, json string of name:value pairs') parser.add_argument('--iterate', help='force iteration', action='store_true') parser.add_argument( '--schema', help='return schema instead, function = [endpoint.method]', action='store_true') # initialize project ( used to load standard credentials parameters ) project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-k', '-v')) # show schema if project.args.schema: endpoint, method = project.args.function.rsplit('.', 1) print(json.dumps(Discovery_To_BigQuery(project.args.api, project.args.version).method_schema(endpoint, method), indent=2, default=str)) # or fetch results else: # the api wrapper takes parameters as JSON job = { 'auth': 'service' if project.args.service else 'user', 'api': project.args.api, 'version': project.args.version, 'function': project.args.function, 'key': project.args.key, 'uri': project.args.uri, 'kwargs': json.loads(project.args.kwargs), 'headers': {}, 'iterate': project.args.iterate, } if project.args.developer_token: job['headers']['developer-token'] = project.args.developer_token if project.args.login_customer_id: job['headers']['login-customer-id'] = project.args.login_customer_id # run the API call results = API(job).execute() # display results if project.args.iterate: for result in results: pprint.PrettyPrinter().pprint(result) else: pprint.PrettyPrinter().pprint(results)