def test_file_credentials_user(self):
config = Configuration(user=self.user_file)
service = get_service(config, 'oauth2', 'v2', 'user')
response = service.userinfo().get().execute()
self.assertIn('email', response)
self.helper_refresh()
def main():
# load standard parameters
parser = commandline_parser()
parser.add_argument(
'--recipe_out',
'-rc',
help='Path to recipe file to be written if replacing fields.',
default=None)
args = parser.parse_args()
# load json to get each task
recipe = get_recipe(args.json)
# check if all fields have been converted to values
validate(recipe, args.no_input)
# check to write converted fields to stdout
if args.recipe_out:
print()
print('Writing to:', args.recipe_out)
f = open(args.recipe_out, 'w')
f.write(json.dumps(recipe, sort_keys=True, indent=2))
f.close()
exit()
# initialize the project singleton with passed in parameters
configuration = Configuration(recipe, args.project, args.user,
args.service, args.client, args.json,
args.key, args.verbose, args.trace_print,
args.trace_file)
execute(configuration, recipe['tasks'], args.force, args.instance)
def setUp(self):
self.user_file = '/tmp/test_user.json'
self.service_file = '/tmp/test_service.json'
self.config = Configuration(service=self.service_file,
user=self.user_file)
shutil.copyfile(UI_USER, self.user_file)
shutil.copyfile(UI_SERVICE, self.service_file)
def test_remote_credentials_user(self):
config = Configuration(user=self.user_file)
credentials = get_credentials(config, 'user')
account = Account.objects.get_or_create_user(credentials, 'password')
clear_credentials_cache()
config = Configuration(user=account.get_credentials_path())
self.assertEqual(config.recipe['setup']['auth']['user'],
account.get_credentials_path())
service = get_service(config, 'oauth2', 'v2', 'user')
response = service.userinfo().get().execute()
self.assertIn('email', response)
self.helper_refresh()
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to transform excel sheets into csv files.
Prints to STDOUT, user is expected to pipe output into file.
Typically used for BigQuery data imports.
Examples:
List sheets in workbook: python helper.py [EXCEL FILE] --list
Convert excel to CSV: python helper.py [EXCEL FILE] --sheet [SHEET NAME] > results.csv
"""))
parser.add_argument('workbook', help='name of file to pull the rows.')
parser.add_argument('--sheet', help='Sheet to pull the rows.', default=None)
parser.add_argument('--list', help='List reports.', action='store_true')
# initialize project
parser = commandline_parser(parser, arguments=('-v'))
args = parser.parse_args()
config = Configuration(
verbose=args.verbose
)
with open(args.workbook, 'rb') as excel_file:
if args.list:
for sheet in excel_to_sheets(excel_file):
print(sheet)
elif args.sheet:
for sheet, row in excel_to_rows(excel_file, args.sheet):
print(rows_to_csv(row).read())
def test_dictionary_credentials_service(self):
with open(self.service_file, 'r') as json_file:
config = Configuration(service=json.load(json_file))
service = get_service(config, 'cloudresourcemanager', 'v1', 'service')
response = service.projects().list().execute()
self.assertIn('projects', response)
def group_instances_resize(count):
return API_Compute(Configuration(service=settings.UI_SERVICE,
project=settings.UI_PROJECT),
auth='service').instanceGroupManagers().resize(
project=settings.UI_PROJECT,
zone=settings.UI_ZONE,
instanceGroupManager='starthinker-worker-group',
size=count).execute()
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to send email template via gMail.
Email templates are JSON that assembles into both HTMl and TXT parts of an email.
For email sample see: https://github.com/google/starthinker/blob/master/starthinker/task/newsletter/sample.json
Example:
- Generate an HTML page from a template, then view via browser.
python newsletter.py --template scripts/newsletter_sample.json > ~/Downloads/email.html
- Send an email template via gMail.
python newsletter.py --template scripts/newsletter_sample.json --email_to [email protected] --email_from [email protected] -u $STARTHINKER_USER
"""))
# get parameters
parser.add_argument(
'--template',
help='template to use for email',
default=None,
required=True)
parser.add_argument('--email_to', help='email to', default=None)
parser.add_argument('--email_from', help='email from', default=None)
# initialize project
parser = commandline_parser(parser, arguments=('-u', '-c', '-v'))
args = parser.parse_args()
config = Configuration(
user=args.user,
client=args.client,
verbose=args.verbose
)
# load template
with open(args.template, 'r') as json_file:
email = EmailTemplate(json.load(json_file))
# send or print
if args.email_to and args.email_from:
print('EMAILING: ', args.email_to)
send_email(
config,
'user',
args.email_to,
args.email_from,
None,
email.get_subject(),
email.get_text(),
email.get_html()
)
else:
# write to STDOUT
print(email.get_html())
print('<pre style="width:600px;margin:0px auto;">%s</pre>' % email.get_text())
def test_dictionary_credentials_user(self):
with open(self.user_file, 'r') as json_file:
config = Configuration(user=json.load(json_file))
service = get_service(config, 'oauth2', 'v2', 'user')
response = service.userinfo().get().execute()
self.assertIn('email', response)
self.helper_refresh()
def account_create():
accounts = Account.objects.all()
if len(accounts) > 0:
account = accounts[0]
else:
config = Configuration(client=UI_CLIENT, user=UI_USER)
credentials = get_credentials(config, 'user')
account = Account.objects.get_or_create_user(credentials, 'password')
return account
def group_instances_list(statuses=[]):
return API_Compute(
Configuration(service=settings.UI_SERVICE,
project=settings.UI_PROJECT),
auth='service',
iterate=True).instanceGroupManagers().listManagedInstances(
project=settings.UI_PROJECT,
zone=settings.UI_ZONE,
instanceGroupManager='starthinker-worker-group',
filter=' OR '.join(
['(instanceStatus = "%s")' % status for status in statuses]),
orderBy='creationTimestamp desc').execute()
def group_instances_delete(name):
return API_Compute(
Configuration(service=settings.UI_SERVICE,
project=settings.UI_PROJECT),
auth='service').instanceGroupManagers().deleteInstances(
project=settings.UI_PROJECT,
zone=settings.UI_ZONE,
instanceGroupManager='starthinker-worker-group',
body={
'instances':
['zones/%s/instances/%s' % (settings.UI_ZONE, name)],
'type': 'PROACTIVE'
}).execute()
def python_task(self, function, instance):
print('STARTHINKER TASK:', function, instance)
PythonOperator(task_id='%s_%d' % (function, instance),
python_callable=getattr(
import_module('starthinker.task.%s.run' % function),
function),
op_kwargs={
'config': Configuration(recipe=self.recipe),
'tasks': self.recipe['tasks'][instance - 1]
},
dag=self.dag)
return self
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Creates USER credentials from Google Cloud Project CLIENT Credentials and displays profile information if it worked.
CLIENT credentials are required to run this script, to obtain the JSON file...
Step 1: Configure Authentication Consent Screen ( do only once )
----------------------------------------
A. Visit: https://console.developers.google.com/apis/credentials/consent
B. Choose Internal if you have GSuite, otherwise choose External.
C. For Application Name enter: StarThinker
D. All other fields are optional, click Save.
Step 2: Create CLIENT Credentials ( do only once )
----------------------------------------
A. Visit: https://console.developers.google.com/apis/credentials/oauthclient
B. Choose Desktop.
C. For Name enter: StarThinker.
D. Click Create and ignore the confirmation pop-up.
Step 3: Download CLIENT Credentials File ( do only once )"
----------------------------------------"
A. Visit: https://console.developers.google.com/apis/credentials"
B. Find your newly created key under OAuth 2.0 Client IDs and click download arrow on the right."
C. The downloaded file is the CLIENT credentials, use its path for the --client -c parameter.
Step 4: Generate USER Credentials File ( do only once )"
----------------------------------------"
A. Run this command with parameters -c [CLIENT file path] and -u [USER file path].
B. The USER file will be created and can be used to access Google APIs.
C. The user profile will be printed to the screen
Example: python helper.py -u [CLIENT file path] -c [USER file path]
"""))
# initialize project
parser = commandline_parser(parser, arguments=('-c', '-u'))
args = parser.parse_args()
config = Configuration(
user=args.user,
client=args.client
)
# get profile to verify everything worked
print('Profile:', json.dumps(get_profile(config), indent=2, sort_keys=True))
def log_get(recipe_id=[], timezone='America/Los_Angeles', days=1):
"""Returns last actionable job run for a specific recipe or all recipes.
Pulls status entries from StackDriver in reverse order. A single recipe may
be run multiple times for multiple tasks at different hours, do not
assume a JOB_END means a recipe is complete. Only way to ensure a recipe is
complete
is to compare all tasks run against all tasks in recipe ( not done by log
code).
Args: - recipe_id ( string or list ) - Optional, if provided returns a single
record for a single job. - timezone ( string ) - The local timezone to cast
all record times into.
Returns:
- ( iterator ) - Each log entry.
"""
body = {
'resourceNames': [
'projects/%s' % UI_PROJECT,
],
'filter':
'\
logName="projects/%s/logs/StarThinker" \
AND labels.version="%s" \
AND labels.layer="JOB" \
' % (UI_PROJECT, LOG_VERSION),
'orderBy':
'timestamp desc',
'pageSize':
1000
}
if recipe_id:
if isinstance(recipe_id, str):
recipe_id = [recipe_id]
body['filter'] += ' AND ( %s )' % ' OR '.join('operation.id="%s"' % r
for r in recipe_id)
for entry in API_StackDriver(
Configuration(service=UI_SERVICE, project=UI_PROJECT),
'service',
iterate=True).entries().list(body=body).execute():
yield entry
def handle(self, *args, **kwargs):
user = None
accounts = Account.objects.all()
# find user to send from
for account in accounts:
if account.email == kwargs['email_from']:
user = account.get_credentials_path()
if user:
print('SEND USER FOUND')
# load template
with open(kwargs['template'], 'r') as json_file:
email = EmailTemplate(json.load(json_file))
# loop through accounts
for account in accounts:
# if account is given only do that one
if kwargs['email_to'] is None or account.email == kwargs[
'email_to']:
if account.email in kwargs['ignore']:
print('IGNORING: ', account.email)
else:
print('EMAILING: ', account.email)
if kwargs['test']:
# write to STDOUT
print(email.get_html())
else:
# send message via email
send_email(Configuration(user=user), 'user',
account.email, kwargs['email_from'],
None, email.get_subject(),
email.get_text(), email.get_html())
sleep(1)
def test_file_credentials_service(self):
config = Configuration(service=self.service_file)
service = get_service(config, 'cloudresourcemanager', 'v1', 'service')
response = service.projects().list().execute()
self.assertIn('projects', response)
def handle(self, *args, **kwargs):
# loop through accounts
for account in Account.objects.all():
# if account is given only do that one
if kwargs['email'] is None or account.email == kwargs['email']:
print('CHECKING: ', account.email)
status = False
# start an email template
email = EmailTemplate()
email.segment_next()
email.greeting(account.name)
email.paragraph(EMAIL_PARAGRAPH)
# loop through recipes
rows = []
for recipe in account.recipe_set.all():
log = recipe.get_log()
rows.append([
recipe.name,
log.get('status'),
log.get('ago'),
'http://starthinker.corp.google.com/recipe/edit/%d/' %
recipe.pk
])
if rows:
email.segment_next()
email.header('Recipe Status')
email.table(RECIPE_SCHEMA, rows)
status = True
# loop through storage
#rows = []
#for recipe in storage_list(account):
# recipe.log = logs.get(recipe.uid(), {})
# rows.append([recipe.name, recipe.log.get('status'), recipe.log.get('time_ago'), recipe.link_storage])
if rows:
email.segment_next()
email.header('Storage Status')
email.table(RECIPE_SCHEMA, rows)
status = True
# # if at least one solution or recipe is running....
# if status:
#
# # show one random recipe
# email.segment_next()
# email.header('Featured Recipes')
# email.paragraph('Each week we feature three recipes that could help your client or agency project. This weeks featured recipe is...')
#
# # script: card ( fetch random )
# for s in sample(list(Script.get_scripts()), 3):
# email.header(s.get_name())
# email.paragraph(s.get_description())
#
# # solution pitch
# email.paragraph('Benefits', bold=True)
# email.list(s.get_pitches())
#
# # solution impact
# email.table(IMPACT_SCHEMA, [(i[0], '%d%%' % i[1]) for i in s.get_impacts().items()])
#
# email.table(DETAILS_SCHEMA, [
# ('Requires', ', '.join([r[0] for r in s.get_requirements().items() if r[1]])),
# ('Categories', ', '.join(s.get_categories())),
# ('Authors', mailto(s.get_authors()))
# ])
#
# email.button('Launch %s' % s.get_name(), '%s/client/edit/' % settings.CONST_URL, big=True)
#
# if s.get_open_source():
# email.button('Avaliable As Open Source', s.get_open_source(), big=True)
#
# # loop through links
# email.segment_next()
# email.header('Helpful Links')
# email.paragraph('Reduce solution delivery to minutes and create custom solutions that exceed clients expectations with these helpful guides and tips.')
for h in HELP_LINKS:
email.section(h['name'], h['description'], None,
h['button']['link'], h['button']['text'])
if kwargs['test']:
# write to STDOUT
print(email.get_html())
else:
print('EMAILING: ', account.email)
# send message via email
send_email(
Configuration(user=account.get_credentials_path()),
'user', account.email, EMAIL_FROM, EMAIL_CC,
EMAIL_SUBJECT, email.get_text(), email.get_html())
sleep(3)
def main():
# get parameters
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to get table schema from BigQuery.
Helps developers upload data to BigQuery and pull schemas. These are the
most common BigQuery tasks when developing solutions.
Examples:
Display table schema: `python helper.py --project [id] --dataset [name] --table [name] -s [credentials]`
Upload csv table: `python helper.py --project [id] --dataset [name] --table [name] --csv [file] --schema [file] -s [credentials]`
Upload excel sheet: `python helper.py --project [id] --dataset [name] --table [name] --excel_file [file] --excel_sheet [name] --schema [file] -s [credentials]`
"""))
parser.add_argument('--dataset',
help='name of BigQuery dataset',
default=None)
parser.add_argument('--table', help='name of BigQuery table', default=None)
parser.add_argument('--csv', help='CSV file path', default=None)
parser.add_argument('--schema', help='SCHEMA file path', default=None)
parser.add_argument('--excel_workbook',
help='Excel file path',
default=None)
parser.add_argument('--excel_sheet', help='Excel sheet name', default=None)
# initialize project
parser = commandline_parser(parser,
arguments=('-u', '-c', '-s', '-v', '-p'))
args = parser.parse_args()
config = Configuration(user=args.user,
client=args.client,
service=args.service,
verbose=args.verbose,
project=args.project)
auth = 'service' if args.service else 'user'
schema = json.loads(args.schema) if args.schema else None
if args.csv:
with open(args.csv, 'r') as csv_file:
rows = csv_to_rows(csv_file.read())
if not schema:
rows, schema = get_schema(rows)
print('DETECETED SCHEMA', json.dumps(schema))
print('Please run again with the above schema provided.')
exit()
rows_to_table(config, auth, config.project, args.dataset,
args.table, rows, schema)
elif args.excel_workbook and args.excel_sheet:
with open(args.excel_workbook, 'r') as excel_file:
rows = excel_to_rows(excel_file, args.excel_sheet)
if not schema:
rows, schema = get_schema(rows)
print('DETECETED SCHEMA', json.dumps(schema))
print('Please run again with the above schema provided.')
exit()
rows_to_table(config, auth, config.project, args.dataset,
args.table, rows, schema)
else:
# print schema
print(
json.dumps(table_to_schema(config, auth, config.project,
args.dataset, args.table),
indent=2))
def run(request):
recipe = request.get_json(force=True)
execute(Configuration(recipe=recipe, verbose=True),
recipe['tasks'],
force=True)
return 'DONE'
def ready(self):
print('CHECKING IF USER BUCKET EXISTS:', USER_BUCKET, USER_LOCATION)
bucket_create(
Configuration(service=settings.UI_SERVICE,
project=settings.UI_PROJECT), 'service',
settings.UI_PROJECT, USER_BUCKET, USER_LOCATION)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line interface for running Google API calls. Any API works. Allows developers to quickly test
and debug API calls before building them into scripts. Useful for debugging permission or call errors.
Examples:
- Pull a DBM report via API.
- https://developers.google.com/bid-manager/v1/queries/getquery
- python google_api.py -api doubleclickbidmanager -version v1 -function queries.getquery -kwargs '{ "queryId": 132865172 }' -u [credentials path]
- Pull a list of placements:
- https://developers.google.com/doubleclick-advertisers/v3.3/placements/list
- python google_api.py -api dfareporting -version v3.3 -function placements.list -kwargs '{ "profileId":2782211 }' -u [credentials path]
- Show schema for Campaign Manager advertiser list endpoint.
- https://developers.google.com/doubleclick-advertisers/v3.4/advertisers/list
- python google_api.py -api dfareporting -version v3.4 -function advertisers.list --schema
- python google_api.py -api dfareporting -version v3.4 -function Advertiser --object
- python google_api.py -api dfareporting -version v3.4 -function Advertiser --struct
"""))
# get parameters
parser.add_argument('-api', help='api to run, name of product api')
parser.add_argument('-version', help='version of api')
parser.add_argument('-function',
help='function or resource to call in api')
parser.add_argument('-uri', help='uri to use in api', default=None)
parser.add_argument('-developer-token',
help='developer token to pass in header',
default=None)
parser.add_argument(
'-login-customer-id',
help='customer to log in with when manipulating an MCC',
default=None)
parser.add_argument(
'-kwargs',
help='kwargs to pass to function, json string of name:value pairs')
parser.add_argument('--iterate',
help='force iteration',
action='store_true')
parser.add_argument('--limit',
type=int,
help='optional, number of records to return',
default=None)
parser.add_argument(
'--schema',
help='return function as BigQuery schema, function = [endpoint.method]',
action='store_true')
parser.add_argument(
'--object',
help=
'return resource as JSON discovery document, function = [resource]',
action='store_true')
parser.add_argument(
'--struct',
help='return resource as BigQuery structure, function = [resource]',
action='store_true')
# initialize project
parser = commandline_parser(parser,
arguments=('-u', '-c', '-s', '-k', '-v'))
args = parser.parse_args()
config = Configuration(user=args.user,
client=args.client,
service=args.service,
key=args.key,
verbose=args.verbose)
# show schema
if args.object:
print(
json.dumps(Discovery_To_BigQuery(
args.api, args.version).resource_json(args.function),
indent=2,
default=str))
elif args.struct:
print(
Discovery_To_BigQuery(args.api,
args.version).resource_struct(args.function))
# show schema
elif args.schema:
print(
json.dumps(Discovery_To_BigQuery(
args.api, args.version).method_schema(args.function),
indent=2,
default=str))
# or fetch results
else:
# the api wrapper takes parameters as JSON
job = {
'auth': 'service' if args.service else 'user',
'api': args.api,
'version': args.version,
'function': args.function,
'key': args.key,
'uri': args.uri,
'kwargs': json.loads(args.kwargs),
'headers': {},
'iterate': args.iterate,
'limit': args.limit,
}
if args.developer_token:
job['headers']['developer-token'] = args.developer_token
if args.login_customer_id:
job['headers']['login-customer-id'] = args.login_customer_id
# run the API call
results = API(job).execute()
# display results
if args.iterate:
for result in results:
pprint.PrettyPrinter().pprint(result)
else:
pprint.PrettyPrinter().pprint(results)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to help debug CM reports and build reporting tools.
Examples:
To get list of reports: python cm.py --account [id] --list -u [user credentials path]
To get report: python cm.py --account [id] --report [id] -u [user credentials path]
To get report files: python cm.py --account [id] --files [id] -u [user credentials path]
To get report sample: python cm.py --account [id] --sample [id] -u [user credentials path]
To get report schema: python cm.py --account [id] --schema [id] -u [user credentials path]
"""))
parser.add_argument('--account',
help='Account ID to use to pull the report.',
default=None)
parser.add_argument('--report',
help='Report ID to pull JSON definition.',
default=None)
parser.add_argument('--schema',
help='Report ID to pull achema definition.',
default=None)
parser.add_argument('--sample',
help='Report ID to pull sample data.',
default=None)
parser.add_argument('--files',
help='Report ID to pull file list.',
default=None)
parser.add_argument('--list', help='List reports.', action='store_true')
# initialize project
parser = commandline_parser(parser, arguments=('-u', '-c', '-s', '-v'))
args = parser.parse_args()
config = Configuration(user=args.user,
client=args.client,
service=args.service,
verbose=args.verbose)
auth = 'service' if args.service else 'user'
is_superuser, profile = get_profile_for_api(config, auth, args.account)
kwargs = {
'profileId': profile,
'accountId': args.account
} if is_superuser else {
'profileId': profile
}
# get report list
if args.report:
kwargs['reportId'] = args.report
report = API_DCM(
config, auth,
internal=is_superuser).reports().get(**kwargs).execute()
print(json.dumps(report, indent=2, sort_keys=True))
# get report files
elif args.files:
kwargs['reportId'] = args.files
for rf in API_DCM(
config, auth, internal=is_superuser,
iterate=True).reports().files().list(**kwargs).execute():
print(json.dumps(rf, indent=2, sort_keys=True))
# get schema
elif args.schema:
filename, report = report_file(config, auth, args.account, args.schema,
None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
print(json.dumps(report_schema(next(rows)), indent=2, sort_keys=True))
# get sample
elif args.sample:
filename, report = report_file(config, auth, args.account, args.sample,
None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
for r in rows_print(rows, row_min=0, row_max=20):
pass
# get list
else:
for report in API_DCM(config,
auth,
internal=is_superuser,
iterate=True).reports().list(**kwargs).execute():
print(json.dumps(report, indent=2, sort_keys=True))
def log_put(event, severity, job=None, text=None, payload=None):
"""Generic log writer used by helper functions.
Writes to StackDriver.
Creates a record that can be read using log_get function. Entire recipe is
logged, worker data and stdout and stderr are added to the JOSN under worker
key.
Only JOB_EXCEPTION and MANAGER_EXCEPTION logs to text in case JSON is corrupt,
everythng else is JSON.
Do not call this directly, use helper functions instead:
- log_manager_start
- log_manager_timeout
- log_manager_error
- log_manager_end
- log_job_dispatch
- log_job_receive
- log_job_start
- log_job_end
- log_job_error
- log_job_timeout
WARNING: Do not corrupt recipe in log code, it is actively being used by
workers while being logged.
Args:
- event ( string ): One of the JOB_* enums.
- severity ( enum ): Stackdriver severity level.
- job ( json ): Recipe workflow to execute.
- text ( string ): Messaging output form the task. Usual stdout and stderr.
- payload ( json ): Output from the scaler or any generic json payload.
"""
if VERBOSE:
print('LOGGING:', event, severity, text or '')
body = {
'entries': [{
'logName': 'projects/%s/logs/StarThinker' % UI_PROJECT,
'severity': severity,
'resource': {
'type': 'project',
'labels': {
'key': UI_PROJECT
},
},
'labels': {
'version': LOG_VERSION,
'layer': event.split('_')[0],
'event': event,
'instance': get_instance_name(),
},
#"operation": {
# "id": string
# "producer": string
# "first": False,
# "last": False,
#},
# already in recipe worker logging task and instance, does this have additional value?
#"sourceLocation": {
# "file": string,
# "line": string,
# "function": string
#},
}],
'partialSuccess':
False,
'dryRun':
False
}
if text is not None:
body['entries'][0]['textPayload'] = text
elif payload is not None:
body['entries'][0]['jsonPayload'] = payload
else:
# Removing tasks from job REMOVES ALL POTENTIAL CREDENTIALS IN CODE
job_buffer = json.loads(
json.dumps(job, indent=2, sort_keys=True, default=str))
if 'tasks' in job_buffer['recipe']:
del job_buffer['recipe']['tasks']
if 'auth' in job_buffer['recipe']['setup']:
del job_buffer['recipe']['setup']['auth']
body['entries'][0]['jsonPayload'] = job_buffer
try:
API_StackDriver(Configuration(service=UI_SERVICE, project=UI_PROJECT),
'service').entries().write(body=body).execute()
except:
print('LOG EVENT ERROR')
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to help debug DV360 reports and build reporting tools.
Examples:
To get list of reports: python dv.py --list -u [user credentials path]
To get report json: python dv.py --report [id] -u [user credentials path]
To get report schema: python dv.py --schema [id] -u [user credentials path]
To get report sample: python dv.py --sample [id] -u [user credentials path]
"""))
# create parameters
parser.add_argument('--report',
help='report ID to pull json definition',
default=None)
parser.add_argument('--schema',
help='report ID to pull schema format',
default=None)
parser.add_argument('--sample',
help='report ID to pull sample data',
default=None)
parser.add_argument('--list', help='list reports', action='store_true')
# initialize project
parser = commandline_parser(parser, arguments=('-u', '-c', '-s', '-v'))
args = parser.parse_args()
config = Configuration(user=args.user,
client=args.client,
service=args.service,
verbose=args.verbose)
auth = 'service' if args.service else 'user'
# get report
if args.report:
report = API_DBM(
config, auth).queries().getquery(queryId=args.report).execute()
print(json.dumps(report, indent=2, sort_keys=True))
# get schema
elif args.schema:
filename, report = report_file(config, auth, args.schema, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
print(json.dumps(get_schema(rows)[1], indent=2, sort_keys=True))
# get sample
elif args.sample:
filename, report = report_file(config, auth, args.sample, None, 10)
rows = report_to_rows(report)
rows = report_clean(rows)
rows = rows_to_type(rows)
for r in rows_print(rows, row_min=0, row_max=20):
pass
# get list
else:
for report in API_DBM(config, auth,
iterate=True).queries().listqueries().execute():
print(json.dumps(report, indent=2, sort_keys=True))
