def script_interactive():
args = {}
from_json = sys.argv[1]
to_json = sys.argv[2] if len(sys.argv) == 3 else ''
script = get_project(sys.argv[1])
# parse fields and constants into parameters
fields = json_get_fields(script)
if to_json:
print('\n(1 of %d) From %s template create recipe: %s\n' %
(len(fields), from_json, to_json))
else:
print('\n(1 of %d) Recipe file to create from %s template.\n' %
(len(fields), sys.argv[1]))
to_json = input("Full Path TO JSON File:")
for count, field in enumerate(fields):
print(
'\n(%d of %d) %s' %
(count + 2, len(fields), field['description']), )
if 'default' in field:
print(' ( Default to "%s" if blank. )' % field['default'], )
print('\n')
args[field['name']] = input("%s ( %s ):" %
(field['name'], field['kind']))
print('\n')
# remove blanks ( they should have defaults )
if not args[field['name']]: del args[field['name']]
script_write(script, args, to_json)
def load_scripts():
if not SCRIPTS:
for root, dirs, files in os.walk(UI_ROOT + '/scripts/'):
for filename in files:
if filename.endswith('.json'):
try:
script = get_project(root + filename)
if not 'script' in script: continue
script['path'] = root + filename
SCRIPTS[filename.replace('script_', '', 1).replace('.json', '', 1)] = script
print('OK', filename)
except Exception as e:
print('ERROR:', filename, str(e))
def script_test_include():
script = get_project(sys.argv[1])
# parse fields and constants into parameters
print(' { "include":{')
print(' "script":"%s",' % sys.argv[1])
print(' "parameters":{')
print(',\n'.join([
' "%s":{"field":{ "name":"%s", "kind":"%s", "description":"%s" }}'
% (field['name'], field['name'], field['kind'],
field.get('description', '')) for field in json_get_fields(script)
]))
print(' }')
print(' }}')
def load_scripts():
if not SCRIPTS:
for root, dirs, files in os.walk(UI_ROOT + '/starthinker/'):
for filename in files:
if RE_SCRIPT.match(filename):
try:
script = get_project(root + '/' + filename)
if not 'script' in script: continue
script['path'] = root + '/' + filename
SCRIPTS[filename.replace('script_', '',
1).replace('.json', '',
1)] = script
print 'OK', filename
except Exception, e:
print 'ERROR:', filename, str(e)
def json_expand_includes(script):
expanded_tasks = []
for task in script['tasks']:
function, parameters = next(iter(task.items()))
if function == 'include':
tasks = get_project(UI_ROOT + '/' + parameters['script'])['tasks']
json_set_fields(tasks, parameters['parameters'])
for t in tasks:
function, parameters = next(iter(t.items()))
expanded_tasks.append({function: parameters})
else:
expanded_tasks.append({function: parameters})
script['tasks'] = expanded_tasks
return script
def script_commandline():
script = get_project(sys.argv[1])
# assemble parameters
parser = argparse.ArgumentParser()
parser.add_argument('json',
help='JSON recipe template to configure.',
default=None)
# parse fields and constants into parameters
for field in json_get_fields(script):
parser_add_field(parser, field)
# run new arg parser with parameters from script
args = vars(parser.parse_args())
# always write to STDOUT, caller whould rediect output to a JSON file
script_write(script, args)
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Evaluate the validity of a json file. Helps in debugging recipes.
Print the line and character position of any errors in the given json file.
Example: python helper.py scripts/say_hello.json
"""))
parser.add_argument('file', help='A JSON file.')
args = parser.parse_args()
try:
project = get_project(args.file)
print('JSON OK:', args.file)
except JSONDecodeError as e:
print(str(e))
def main():
# this is a helper function, these inputs mirror util.project.Project singleton used by tasks because they are pass through to each task
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Command line to execute all tasks in a recipe once. ( Common Entry Point )
This script dispatches all the tasks in a JSON recipe to handlers in sequence.
For each task, it calls a subprocess to execute the JSON instructions, waits
for the process to complete and dispatches the next task, until all tasks are
complete or a critical failure ( exception ) is raised.
If an exception is raised in any task, all following tasks are not executed by design.
Example: python run.py [path to recipe file] --force
Caution: This script does NOT check if the last job finished, potentially causing overruns.
Notes:
- To avoid running the entire script when debugging a single task, the command line
can easily replace "all" with the name of any "task" in the json. For example
python all/run.py project/sample/say_hello.json
- Can be easily replaced with the following to run only the "hello" task:
python task/hello/run.py project/sample/say_hello.json
- Or specified further to run only the second hello task:
python task/hello/run.py project/sample/say_hello.json -i 2
"""))
parser.add_argument('json', help='path to tasks json file')
parser.add_argument('--project',
'-p',
help='cloud id of project, defaults to None',
default=None)
parser.add_argument(
'--user',
'-u',
help=
'path to user credentials json file, defaults to GOOGLE_APPLICATION_CREDENTIALS',
default=None)
parser.add_argument(
'--service',
'-s',
help='path to service credentials json file, defaults None',
default=None)
parser.add_argument(
'--client',
'-c',
help='path to client credentials json file, defaults None',
default=None)
parser.add_argument('--verbose',
'-v',
help='print all the steps as they happen.',
action='store_true')
parser.add_argument(
'--date',
'-d',
help=
'YYYY-MM-DD format date for which these reports are to be run, default will be today.',
default='TODAY')
parser.add_argument('--force',
'-force',
help='execute all scripts once then exit.',
action='store_true')
parser.add_argument(
'--trace_print',
'-tp',
help='Simplified execution trace of the program written to stdout.',
action='store_true')
parser.add_argument(
'--trace_file',
'-tf',
help='Simplified execution trace of the program written to file.',
action='store_true')
args = parser.parse_args()
# load json to get each task
recipe = get_project(args.json)
# track per task instance count
instances = {}
returncode = 0
for task in recipe['tasks']:
script, task = next(iter(task.items()))
# count instance per task
instances.setdefault(script, 0)
instances[script] += 1
# assemble command ( replace command, use all arguments passed, and add instance )
command = 'python -W ignore %s/starthinker/task/%s/run.py %s -i %d' % (
UI_ROOT, script, ' '.join(sys.argv[1:]), instances[script])
# show command so user can run each task
print(command)
# execute command if schedule, return code
if args.force or is_scheduled(recipe, task):
returncode |= subprocess.Popen(command, shell=True).wait()
# skip command if not schedule
else:
raise SystemExit(
'Schedule Skipping: run command manually or add --force to run all'
)
# Set lowest return code from all tasks
exit(returncode)
###########################################################################
"""Evaluate the validity of a json file. Helps in debugging recipes.
Print the line and character position of any errors in the given json file.
Arguments
file - path to JSON file to be evaluated
Example
python project/helper.py project/sample.json
"""
import argparse
from starthinker.util.project import get_project
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('file', help='A JSON file.')
args = parser.parse_args()
try:
project = get_project(args.file, debug=True)
print 'JSON OK:', args.file
except Exception, e:
print 'JSON ERROR:', args.file, str(e)
"""Evaluate the validity of a json file. Helps in debugging recipes.
Print the line and character position of any errors in the given json file.
Arguments
file - path to JSON file to be evaluated
Example
python project/helper.py project/sample.json
"""
import argparse
from json import JSONDecodeError
from starthinker.util.project import get_project
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('file', help='A JSON file.')
args = parser.parse_args()
try:
project = get_project(args.file)
print('JSON OK:', args.file)
except JSONDecodeError as e:
print(str(e))
def __init__(self, _recipe_path):
self.recipe_path = _recipe_path
self.recipe = get_project(_recipe_path)
self.dag = None
help='print all the steps as they happen.',
action='store_true')
parser.add_argument(
'--date',
'-d',
help=
'YYYY-MM-DD format date for which these reports are to be run, default will be today.',
default='TODAY')
parser.add_argument('--force',
'-force',
help='execute all scripts once then exit.',
action='store_true')
args = parser.parse_args()
# load json to get each task
project = get_project(args.json)
# track per task instance count
instances = {}
# return code changes if a task fails
return_code = 0
for task in project['tasks']:
script, task = task.items()[0]
# count instance per task
instances.setdefault(script, 0)
instances[script] += 1
# assemble command ( replace command, use all arguments passed, and add instance )
command = 'python %stask/%s/run.py %s -i %d' % (
EXECUTE_PATH, script, ' '.join(sys.argv[1:]), instances[script])
action='store_true')
parser.add_argument('--force',
'-f',
help='execute all scripts once then exit.',
action='store_true')
#parser.add_argument('--remote', '-r', help='execute the scripts remotely, requires pub/sub config.', action='store_true')
args = parser.parse_args()
try:
while True:
for filepath in glob('%s*.json' % args.path):
if args.verbose: print 'PROJECT:', filepath
project = get_project(filepath)
if args.force or is_scheduled(project):
command = 'python task/%s/run.py %s %s' % (
script, filepath, ' '.join(sys.argv[2:]))
if args.verbose: print 'COMMAND:', command
subprocess.Popen(command, shell=True, cwd=EXECUTE_PATH)
if args.force: break
if args.verbose: print 'SLEEP:', ONE_HOUR_AND_ONE_SECOND
sleep(ONE_HOUR_AND_ONE_SECOND)
except KeyboardInterrupt:
Print the line and character position of any errors in the given json file.
Arguments
file - path to JSON file to be evaluated
Example
python project/helper.py project/sample.json
"""
import argparse
from starthinker.util.project import get_project
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('file', help='A JSON file.')
parser.add_argument('--debug', '-d', help='Debug mode, do not scrub newlines.', action='store_true')
args = parser.parse_args()
try:
project = get_project(args.file, debug=args.debug)
print('JSON OK:', args.file)
except Exception as e:
print('JSON ERROR:', args.file, str(e))
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=textwrap.dedent("""\
Evaluate the validity of a json file. Helps in debugging recipes.
Print the line and character position of any errors in the given json file.
Also provide ability to replace values in a recipe with fields and values.
Examples:
- python helper.py scripts/say_hello.json
- python helper.py scripts/dv360_targeting_audit.json -target 'DV_Targeting_Audit.' -value '{dataset}.'
- python helper.py scripts/dv360_targeting_audit.json -target '"service"' -field "auth_bigquery" -order 3 -description "BigQuery read/ write credentials." -default "service" -kind "authentication"
"""))
parser.add_argument('file', help='A JSON file.')
parser.add_argument('-target', help='The string value to replace.')
parser.add_argument('-delete', action='store_false')
parser.add_argument('-value', help='The value to insert.', default='')
parser.add_argument('-field', help='The field name to insert.')
parser.add_argument('-kind',
help='The field kind to insert.',
default='string')
parser.add_argument('-prefix',
help='The field prefix to insert.',
default='')
parser.add_argument('-default',
help='The field default to insert.',
default='')
parser.add_argument('-description',
help='The field description to insert.',
default='')
parser.add_argument('-order',
help='The field order to insert.',
type=int,
default=1)
parser.add_argument('-choices',
help='The field choices to insert.',
default='')
args = parser.parse_args()
try:
get_project(args.file)
print('JSON OK:', args.file)
except JSONDecodeError as e:
print('JSON ERROR:', str(e))
exit
if args.target:
with open(args.file) as recipe_file:
recipe = recipe_file.read()
value = args.value
if args.field:
value = '{"field":{ "name":"%s", "kind":"%s", "order":%d, "default":"%s"' % (
args.field, args.kind, args.order, args.default)
if args.prefix:
value += ', "prefix":"%s"' % args.prefix
if args.choices:
value += ', "choices":[%s]' % args.coices
value += ', "description":"%s" }}' % args.description
elif args.delete:
value = ''
print('REPLACE:', args.target)
print('WITH:', value)
recipe = recipe.replace(args.target, value)
try:
get_project(None, recipe)
print('NEW JSON OK:', args.file)
except JSONDecodeError as e:
print('NEW JSON ERROR:', str(e))
exit
if str(input('Commit change (y/n): ')).lower().strip() == 'y':
with open(args.file, "w") as f:
f.write(recipe)
from django.conf import settings
from starthinker.config import EXECUTE_PATH
from starthinker.script.parse import json_set_fields, json_set_instructions, json_set_description
from starthinker.util.project import get_project
RE_SCRIPT = re.compile(r'^script_.*\.json$')
# cache scripts and authors in memory
AUTHORS = {}
SCRIPTS = {}
for root, dirs, files in os.walk(EXECUTE_PATH):
for filename in files:
if RE_SCRIPT.match(filename):
try:
script = get_project(root + '/' + filename)
if not 'script' in script: continue
SCRIPTS[filename.replace('script_', '').replace('.json',
'')] = script
# authors ( for solution lookup )
container = root.rsplit('/', 1)[-1]
AUTHORS[container] = AUTHORS.get(container, set()) | set(
script['script'].get('authors', []))
print 'OK', filename
except Exception, e:
print 'ERROR:', filename, str(e)
class Script:
@staticmethod
def get_scripts(account_email=None):
def load_tests():
for root, dirs, files in os.walk(TEST_DIRECTORY):
for filename in files:
if filename.endswith('.json'):
print('LOADING', filename)
yield filename, get_project(TEST_DIRECTORY + filename)
parser.add_argument(
'--trace_print',
'-tp',
help='Simplified execution trace of the program written to stdout.',
action='store_true')
parser.add_argument(
'--trace_file',
'-tf',
help='Simplified execution trace of the program written to file.',
action='store_true')
args = parser.parse_args()
# load json to get each task
recipe = get_project(args.json)
# track per task instance count
instances = {}
returncode = 0
for task in recipe['tasks']:
script, task = next(iter(task.items()))
# count instance per task
instances.setdefault(script, 0)
instances[script] += 1
# assemble command ( replace command, use all arguments passed, and add instance )
command = 'python -W ignore %s/starthinker/task/%s/run.py %s -i %d' % (
UI_ROOT, script, ' '.join(sys.argv[1:]), instances[script])