def validate(recipe, no_input):
# query for fields if they exist
fields = json_get_fields(recipe)
# parse fields and constants into parameters
if fields:
if no_input:
raise NameError(
'Edit the recipie and convert these fields into values:\n - %s\n'
% ' \n - '.join('%s: %s' % (f['name'], f['description'])
for f in fields))
else:
parameters = prompt(fields)
json_set_fields(recipe, parameters)
def generate_include(script_file):
script = get_recipe(script_file)
# parse fields and constants into parameters
print(' { "include":{')
print(' "script":"%s",' % script_file)
print(' "parameters":{')
print(',\n'.join([
' "%s":{"field":{ "name":"%s", "kind":"%s", "description":"%s" }}'
% (field['name'], field['name'], field['kind'],
field.get('description', '')) for field in json_get_fields(script)
]))
print(' }')
print(' }}')
print('')
def __init__(self, sequence, script, values, *args, **kwargs):
self.script = script
self.values = values
super(ScriptJsonForm, self).__init__(*args, **kwargs)
self.fields['script_sequence'].initial = sequence
x = script.get_tag()
self.variables = json_get_fields(script.get_tasks())
for variable in self.variables:
if variable['name'].startswith('recipe_'):
continue # skip inputs that come from recipe constants
self.fields[variable['name']] = get_field_kind(variable)
self.fields[variable['name']].initial = values.get(
variable['name'],
variable.get('value', variable.get('default', '')))
self.fields[variable['name']].required = variable.get(
'required', False)
self.fields[variable['name']].help_text = variable.get(
'description', '')
def script_to_dag(dag_name,
title,
description,
instructions,
script,
parameters={}):
return AIRFLOW_TEMPLATE.format(**{
'title':title,
'description':description,
'instructions':' - ' + '\n - '.join(instructions),
'inputs':fields_to_string(
json_get_fields(script),
parameters
),
'recipe':dict_to_string(
script,
skip=('fields',)
),
'dag':dag_name
})
def configure_tests(tests, runs, skips, test_run_id):
"""Initialize the starthinker_assets/tests.json variable harness.
Read all existing tests from tests/scripts/*.json and create a dictionary of
each script and fields. Save that dictionary to a test harness file where
developer can configure tests.
Then read the test harness and create recipe files to that can be run. Write
those files to tests/recipes/*.json for execution in a later step.
Args:
test: List of (filename, json) pairs containing all the tests.
runs: List of test names that will be run, all will run if blank.
skips: List of tests to skip.
test_run_id: String added as a field to each test, used for namespacing.
Returns:
List of JSON recpies, where all fields have values from the test harness.
"""
# Get old fields from the config file
print('UPDATE CONFIG')
old_fields = {}
if os.path.exists(CONFIG_FILE):
with open(CONFIG_FILE, 'r') as f:
old_fields = json.load(f)
# Get new fields from test files and merge in old values
fields = {}
for filename, script in tests:
script_fields = json_get_fields(script)
script_name = filename.split('.')[0]
fields.setdefault(script_name, {})
for field in script_fields:
if field['name'] == 'test_run_id': continue
fields[script_name][field['name']] = old_fields.get(
script_name, {}).get(field['name'], field.get('default', ''))
fields[script_name][
'%s_description' % field['name']] = '(%s) %s' % (field.get(
'kind',
'string'), field.get('description', 'No description.'))
if field['name'] not in old_fields.get(script_name, {}):
print('NEW FIELD ADDED', script_name, field['name'])
# Save field values to config file
if fields:
f = open(CONFIG_FILE, 'w')
f.write(json.dumps(fields, sort_keys=True, indent=2))
f.close()
else:
print('WARNING CONFIGURATION IS EMPTY, CHECK YOUR PATHS!')
# Inject the test run ID to the list of field values that were read from the
# test config file. This is done in memory only, so that concrete test run
# value are replaced every time a test runs.
if test_run_id:
for script in fields:
fields[script]['test_run_id'] = test_run_id
# Create recipe directory
print('GENERATE RECIPES')
os.makedirs(RECIPE_DIRECTORY, exist_ok=True)
# Create recipes from tests
recipes = []
for filename, script in tests:
name = filename.split('.')[0]
if runs and name not in runs:
continue
if name in skips:
continue
# Set config field values into the script
json_set_fields(script, fields.get(name, {}))
# Expand all includes to full recipe
try:
json_expand_includes(script)
with open(RECIPE_DIRECTORY + filename, 'w') as f:
f.write(json.dumps(script, sort_keys=True, indent=2))
recipes.append(filename)
except FileNotFoundError as e:
print('MISSING INLCLUDE')
print(' - TEST: tests/scripts/%s' % filename)
print(' - INCLUDE: ', str(e))
# Create log directory and clear old logs
os.makedirs(LOG_DIRECTORY, exist_ok=True)
# Display instructions
print('')
print('------')
print('------------')
print('------------------------')
print(
'Some tests require custom values. Update the necessary fields for the tests you wish to run.'
)
print('EDIT: ' + CONFIG_FILE)
print('------------------------')
print(
'Some tests require external assets. Join the following group to gain access.'
)
print('VISIT: https://groups.google.com/forum/#!forum/starthinker-assets')
print('------------------------')
print('------------')
print('------')
print('')
sleep(3)
return recipes
def recipe_to_colab(name, description, instructions, tasks, parameters={}, project=None, client_credentials=None, user_credentials=None, service_credentials=None):
""" Converts a JSON recipe into a Jupyter Notebook for Colabs.
Sets up multiple steps to execute recipe:
1. Install starthinker from repository
2. Get Cloud Project ID.
3. Get Client Credentials ( optional if User Credentials exist ).
4. Enter Recipe parameters if fields present.
5. Execute recipe tasks.
Args:
* name: (string) The name of the notebook.
* description: (string) A description fo the recipe.
* instructions: (string) Recipe manual instructions, for example connecting datastudios.
* tasks: (list) The task JSON to execute.
* parameters: (dict) Values for field parameters in tasks, optional.
* project: (string) The GCP project id.
* client_credentials: (string) The GCP Desktop Client Credentials in JSON string.
* user_credentials: (string) Not used, placeholder.
* service_credentials: (string) Not used, placeholder.
Returns:
* (string) Rendered notebook source code to be written to a ipynb file.
"""
colab = Colab(name)
colab.header('1. Install Dependencies')
colab.paragraph(
'First install the libraries needed to execute recipes, this only needs to be done once, then click play.'
)
colab.code('!pip install git+https://github.com/google/starthinker')
colab.header('2. Get Cloud Project ID')
colab.paragraph(
'To run this recipe [requires a Google Cloud Project](https://github.com/google/starthinker/blob/master/tutorials/cloud_project.md), this only needs to be done once, then click play.'
)
#colab.image('Client Project ID', 'https://github.com/google/starthinker/raw/master/tutorials/images/cloud_project.png')
colab.code('CLOUD_PROJECT = \'%s\'' % (project or 'PASTE PROJECT ID HERE'))
colab.code('\nprint("Cloud Project Set To: %s" % CLOUD_PROJECT)')
colab.header('3. Get Client Credentials')
#colab.image('Client Credentials', 'https://github.com/google/starthinker/raw/master/tutorials/images/cloud_client_installed.png')
colab.paragraph(
'To read and write to various endpoints requires [downloading client credentials](https://github.com/google/starthinker/blob/master/tutorials/cloud_client_installed.md), this only needs to be done once, then click play.'
)
colab.code('CLIENT_CREDENTIALS = \'%s\'' % (client_credentials or 'PASTE CLIENT CREDENTIALS HERE'))
colab.code('\nprint("Client Credentials Set To: %s" % CLIENT_CREDENTIALS)')
fields = json_get_fields(tasks)
if fields:
colab.header('4. Enter %s Parameters' % name)
colab.paragraph(description)
colab.list(instructions)
colab.paragraph(
'Modify the values below for your use case, can be done multiple times, then click play.'
)
colab.code('FIELDS = %s' % fields_to_string(fields, parameters))
colab.code('\nprint("Parameters Set To: %s" % FIELDS)')
colab.header('%d. Execute %s' % (5 if fields else 4, name))
colab.paragraph(
'This does NOT need to be modified unless you are changing the recipe, click play.'
)
colab.code('from starthinker.util.configuration import Configuration')
colab.code('from starthinker.util.configuration import execute')
colab.code('from starthinker.util.recipe import json_set_fields')
colab.code('')
colab.code("USER_CREDENTIALS = '/content/user.json'")
colab.code('')
colab.code('TASKS = %s' %
dict_to_string(json_set_auths(tasks, 'user'), skip=('field',)))
colab.code('')
if fields:
colab.code('json_set_fields(TASKS, FIELDS)')
colab.code('')
colab.code('execute(Configuration(project=CLOUD_PROJECT, client=CLIENT_CREDENTIALS, user=USER_CREDENTIALS, verbose=True), TASKS, force=True)')
return colab.render()