コード例 #1
0
ファイル: run.py プロジェクト: ekarthig/starthinker
def google_api_execute(auth, api_call, results, errors, limit=None):

    try:
        rows = API(api_call).execute()

        if results:
            # check if single object needs conversion to rows
            if isinstance(rows, dict):
                rows = [rows]

            rows = map(lambda r: Discovery_To_BigQuery.clean(r), rows)
            put_rows(auth, results, rows)

            if 'bigquery' in results:
                results['bigquery']['disposition'] = 'WRITE_APPEND'

    except HttpError as e:

        if errors:
            rows = [{
                'Error':
                str(e),
                'Parameters': [{
                    'Key': k,
                    'Value': str(v)
                } for k, v in api_call['kwargs'].items()]
            }]
            put_rows(auth, errors, rows)

            if 'bigquery' in errors:
                errors['bigquery']['disposition'] = 'WRITE_APPEND'

        else:
            raise e
コード例 #2
0
ファイル: run.py プロジェクト: Gregorfran/starthinker-gregor
def google_api():
    if project.verbose:
        print('GOOGLE_API', project.task['api'], project.task['version'])

    results = API(project.task).execute()

    put_rows(project.task['auth'], project.task['out'], results)
コード例 #3
0
def google_api():
    if project.verbose:
        print 'GOOGLE_API', project.task['api'], project.task['version']

    results = API(project.task).execute()

    put_rows(
        project.task['auth'], project.task['out'], '%s_%s.json' %
        (project.task['function'].replace('.', '_'), project.date), results)
コード例 #4
0
def google_api_execute(config, auth, api_call, results, errors, limit=None):
    """Execute the actual API call and write to the end points defined.

  The API call is completely defined at this point.
  The results and error definition is optional.

  Args:
    auth (string): either "user" or "service" to make the API call.
    api_call (dict): the JSON for the API call as defined in recipe.
    results (dict): defines where the data will be written
    errors (dict): defines where the errors will be written
    limit (int): Reduce the number of calls ( mostly for debugging )

  Returns (dict):
    None, all data is transfered between API / BigQuery

  Raises:
    ValueError: If a required key in the recipe is missing.
  """

    try:
        rows = API(config, api_call).execute()

        if results:
            # check if single object needs conversion to rows
            if isinstance(rows, dict):
                rows = [rows]

            # check if simple string API results
            elif results.get('bigquery', {}).get('format', 'JSON') == 'CSV':
                rows = [[r] for r in rows]

            rows = map(lambda r: Discovery_To_BigQuery.clean(r), rows)
            put_rows(config, auth, results, rows)

            if 'bigquery' in results:
                results['bigquery']['disposition'] = 'WRITE_APPEND'

    except HttpError as e:

        if errors:
            rows = [{
                'Error':
                str(e),
                'Parameters': [{
                    'Key': k,
                    'Value': str(v)
                } for k, v in api_call['kwargs'].items()]
            }]
            put_rows(config, auth, errors, rows)

            if 'bigquery' in errors:
                errors['bigquery']['disposition'] = 'WRITE_APPEND'

        else:
            raise e
コード例 #5
0
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""\
      Command line interface for running Google API calls.  Any API works.  Allows developers to quickly test
      and debug API calls before building them into scripts.  Useful for debugging permission or call errors.

      Examples:
        - Pull a DBM report via API.
          - https://developers.google.com/bid-manager/v1/queries/getquery
          - python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]

        - Pull a list of placements:
          - https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
          - python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]

  """))

    # get parameters
    parser.add_argument('-api', help='api to run, name of product api')
    parser.add_argument('-version', help='version of api')
    parser.add_argument('-function', help='function to call in api')
    parser.add_argument('-uri', help='function to call in api', default=None)
    parser.add_argument(
        '-kwargs',
        help='kwargs to pass to function, json string of name:value pairs')
    parser.add_argument('--iterate',
                        help='set to true to force iteration',
                        action='store_true')

    # initialize project ( used to load standard credentials parameters )
    project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-v'))

    # the api wrapper takes parameters as JSON
    job = {
        'auth': 'service' if project.args.service else 'user',
        'api': project.args.api,
        'version': project.args.version,
        'function': project.args.function,
        'uri': project.args.uri,
        'kwargs': json.loads(project.args.kwargs),
        'iterate': project.args.iterate,
    }

    # run the API call
    results = API(job).execute()

    # display results
    if project.args.iterate:
        for result in results:
            pprint.PrettyPrinter().pprint(result)
    else:
        pprint.PrettyPrinter().pprint(results)
コード例 #6
0
ファイル: google_api.py プロジェクト: google/starthinker
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""\
      Command line interface for running Google API calls.  Any API works.  Allows developers to quickly test
      and debug API calls before building them into scripts.  Useful for debugging permission or call errors.

      Examples:
        - Pull a DBM report via API.
          - https://developers.google.com/bid-manager/v1/queries/getquery
          - python google_api.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]

        - Pull a list of placements:
          - https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
          - python google_api.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]

        - Show schema for Campaign Manager advertiser list endpoint.
        - https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list
        - python google_api.py -api dfareporting -version v3.4 -function advertisers.list --schema
        - python google_api.py -api dfareporting -version v3.4 -function Advertiser --object
        - python google_api.py -api dfareporting -version v3.4 -function Advertiser --struct

  """))

    # get parameters
    parser.add_argument('-api', help='api to run, name of product api')
    parser.add_argument('-version', help='version of api')
    parser.add_argument('-function',
                        help='function or resource to call in api')
    parser.add_argument('-uri', help='uri to use in api', default=None)
    parser.add_argument('-developer-token',
                        help='developer token to pass in header',
                        default=None)
    parser.add_argument(
        '-login-customer-id',
        help='customer to log in with when manipulating an MCC',
        default=None)
    parser.add_argument(
        '-kwargs',
        help='kwargs to pass to function, json string of name:value pairs')
    parser.add_argument('--iterate',
                        help='force iteration',
                        action='store_true')
    parser.add_argument('--limit',
                        type=int,
                        help='optional, number of records to return',
                        default=None)
    parser.add_argument(
        '--schema',
        help='return function as BigQuery schema, function = [endpoint.method]',
        action='store_true')
    parser.add_argument(
        '--object',
        help=
        'return resource as JSON discovery document, function = [resource]',
        action='store_true')
    parser.add_argument(
        '--struct',
        help='return resource as BigQuery structure, function = [resource]',
        action='store_true')

    # initialize project
    parser = commandline_parser(parser,
                                arguments=('-u', '-c', '-s', '-k', '-v'))
    args = parser.parse_args()
    config = Configuration(user=args.user,
                           client=args.client,
                           service=args.service,
                           key=args.key,
                           verbose=args.verbose)

    # show schema
    if args.object:
        print(
            json.dumps(Discovery_To_BigQuery(
                args.api, args.version).resource_json(args.function),
                       indent=2,
                       default=str))

    elif args.struct:
        print(
            Discovery_To_BigQuery(args.api,
                                  args.version).resource_struct(args.function))

    # show schema
    elif args.schema:
        print(
            json.dumps(Discovery_To_BigQuery(
                args.api, args.version).method_schema(args.function),
                       indent=2,
                       default=str))

    # or fetch results
    else:

        # the api wrapper takes parameters as JSON
        job = {
            'auth': 'service' if args.service else 'user',
            'api': args.api,
            'version': args.version,
            'function': args.function,
            'key': args.key,
            'uri': args.uri,
            'kwargs': json.loads(args.kwargs),
            'headers': {},
            'iterate': args.iterate,
            'limit': args.limit,
        }

        if args.developer_token:
            job['headers']['developer-token'] = args.developer_token

        if args.login_customer_id:
            job['headers']['login-customer-id'] = args.login_customer_id

        # run the API call
        results = API(job).execute()

        # display results
        if args.iterate:
            for result in results:
                pprint.PrettyPrinter().pprint(result)
        else:
            pprint.PrettyPrinter().pprint(results)
コード例 #7
0
    parser.add_argument('-function', help='function to call in api')
    parser.add_argument('-uri', help='function to call in api', default=None)
    parser.add_argument(
        '-kwargs',
        help='kwargs to pass to function, json string of name:value pairs')
    parser.add_argument('--iterate',
                        help='set to true to force iteration',
                        action='store_true')

    # initialize project ( used to load standard credentials parameters )
    project.from_commandline(parser=parser)

    # the api wrapper takes parameters as JSON
    job = {
        "auth": 'service' if project.args.service else 'user',
        "api": project.args.api,
        "version": project.args.version,
        "function": project.args.function,
        "uri": project.args.uri,
        "kwargs": json.loads(project.args.kwargs),
        "iterate": project.args.iterate,
    }

    results = API(job).execute()

    if project.args.iterate:
        for result in results:
            pprint.PrettyPrinter().pprint(result)
    else:
        pprint.PrettyPrinter().pprint(results)
コード例 #8
0
def main():

  parser = argparse.ArgumentParser(
      formatter_class=argparse.RawDescriptionHelpFormatter,
      description=textwrap.dedent("""\
      Command line interface for running Google API calls.  Any API works.  Allows developers to quickly test
      and debug API calls before building them into scripts.  Useful for debugging permission or call errors.

      Examples:
        - Pull a DBM report via API.
          - https://developers.google.com/bid-manager/v1/queries/getquery
          - python google_api/helper.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]

        - Pull a list of placements:
          - https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
          - python task/google_api/helper.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]

        - Show schema for Campaign Manager advertiser list endpoint.
        - https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list
        - python starthinker/task/google_api/helper.py -api dfareporting -version v3.4 -function advertisers.list --schema

  """))

  # get parameters
  parser.add_argument('-api', help='api to run, name of product api')
  parser.add_argument('-version', help='version of api')
  parser.add_argument('-function', help='function to call in api')
  parser.add_argument('-uri', help='uri to use in api', default=None)
  parser.add_argument(
      '-developer-token',
      help='developer token to pass in header',
      default=None)
  parser.add_argument(
      '-login-customer-id',
      help='customer to log in with when manipulating an MCC',
      default=None)
  parser.add_argument(
      '-kwargs',
      help='kwargs to pass to function, json string of name:value pairs')
  parser.add_argument('--iterate', help='force iteration', action='store_true')
  parser.add_argument(
      '--schema',
      help='return schema instead, function = [endpoint.method]',
      action='store_true')


  # initialize project ( used to load standard credentials parameters )
  project.from_commandline(parser=parser, arguments=('-u', '-c', '-s', '-k', '-v'))

  # show schema
  if project.args.schema:
    endpoint, method = project.args.function.rsplit('.', 1)
    print(json.dumps(Discovery_To_BigQuery(project.args.api, project.args.version).method_schema(endpoint, method), indent=2, default=str))

  # or fetch results
  else:

    # the api wrapper takes parameters as JSON
    job = {
      'auth': 'service' if project.args.service else 'user',
      'api': project.args.api,
      'version': project.args.version,
      'function': project.args.function,
      'key': project.args.key,
      'uri': project.args.uri,
      'kwargs': json.loads(project.args.kwargs),
      'headers': {},
      'iterate': project.args.iterate,
    }

    if project.args.developer_token:
      job['headers']['developer-token'] = project.args.developer_token

    if project.args.login_customer_id:
      job['headers']['login-customer-id'] = project.args.login_customer_id

    # run the API call
    results = API(job).execute()

    # display results
    if project.args.iterate:
      for result in results:
        pprint.PrettyPrinter().pprint(result)
    else:
      pprint.PrettyPrinter().pprint(results)