Esempio n. 1
0
def script_interactive():
    args = {}

    from_json = sys.argv[1]
    to_json = sys.argv[2] if len(sys.argv) == 3 else ''

    script = get_project(sys.argv[1])

    # parse fields and constants into parameters
    fields = json_get_fields(script)

    if to_json:
        print('\n(1 of %d) From %s template create recipe: %s\n' %
              (len(fields), from_json, to_json))
    else:
        print('\n(1 of %d) Recipe file to create from %s template.\n' %
              (len(fields), sys.argv[1]))
        to_json = input("Full Path TO JSON File:")

    for count, field in enumerate(fields):
        print(
            '\n(%d of %d) %s' %
            (count + 2, len(fields), field['description']), )
        if 'default' in field:
            print(' ( Default to "%s" if blank. )' % field['default'], )
        print('\n')
        args[field['name']] = input("%s ( %s ):" %
                                    (field['name'], field['kind']))
        print('\n')

        # remove blanks ( they should have defaults )
        if not args[field['name']]: del args[field['name']]

    script_write(script, args, to_json)
Esempio n. 2
0
def load_scripts():
  if not SCRIPTS:
    for root, dirs, files in os.walk(UI_ROOT + '/scripts/'):
      for filename in files:
        if filename.endswith('.json'):
          try:
            script = get_project(root + filename)
            if not 'script' in script: continue
            script['path'] = root + filename
            SCRIPTS[filename.replace('script_', '', 1).replace('.json', '', 1)] = script
            print('OK', filename)
          except Exception as e:
            print('ERROR:', filename, str(e))
Esempio n. 3
0
def script_test_include():
    script = get_project(sys.argv[1])

    # parse fields and constants into parameters
    print('    { "include":{')
    print('      "script":"%s",' % sys.argv[1])
    print('      "parameters":{')
    print(',\n'.join([
        '        "%s":{"field":{ "name":"%s", "kind":"%s", "description":"%s" }}'
        % (field['name'], field['name'], field['kind'],
           field.get('description', '')) for field in json_get_fields(script)
    ]))
    print('      }')
    print('    }}')
Esempio n. 4
0
def load_scripts():
    if not SCRIPTS:
        for root, dirs, files in os.walk(UI_ROOT + '/starthinker/'):
            for filename in files:
                if RE_SCRIPT.match(filename):
                    try:
                        script = get_project(root + '/' + filename)
                        if not 'script' in script: continue
                        script['path'] = root + '/' + filename
                        SCRIPTS[filename.replace('script_', '',
                                                 1).replace('.json', '',
                                                            1)] = script
                        print 'OK', filename
                    except Exception, e:
                        print 'ERROR:', filename, str(e)
Esempio n. 5
0
def json_expand_includes(script):
    expanded_tasks = []
    for task in script['tasks']:
        function, parameters = next(iter(task.items()))

        if function == 'include':
            tasks = get_project(UI_ROOT + '/' + parameters['script'])['tasks']
            json_set_fields(tasks, parameters['parameters'])
            for t in tasks:
                function, parameters = next(iter(t.items()))
                expanded_tasks.append({function: parameters})

        else:
            expanded_tasks.append({function: parameters})

    script['tasks'] = expanded_tasks

    return script
Esempio n. 6
0
def script_commandline():

    script = get_project(sys.argv[1])

    # assemble parameters
    parser = argparse.ArgumentParser()
    parser.add_argument('json',
                        help='JSON recipe template to configure.',
                        default=None)

    # parse fields and constants into parameters
    for field in json_get_fields(script):
        parser_add_field(parser, field)

    # run new arg parser with parameters from script
    args = vars(parser.parse_args())

    # always write to STDOUT, caller whould rediect output to a JSON file
    script_write(script, args)
Esempio n. 7
0
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""\
      Evaluate the validity of a json file. Helps in debugging recipes.
      Print the line and character position of any errors in the given json file.

      Example: python helper.py scripts/say_hello.json
  """))

    parser.add_argument('file', help='A JSON file.')
    args = parser.parse_args()

    try:
        project = get_project(args.file)
        print('JSON OK:', args.file)
    except JSONDecodeError as e:
        print(str(e))
Esempio n. 8
0
def main():

    # this is a helper function, these inputs mirror util.project.Project singleton used by tasks because they are pass through to each task
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""\
      Command line to execute all tasks in a recipe once. ( Common Entry Point )

      This script dispatches all the tasks in a JSON recipe to handlers in sequence.
      For each task, it calls a subprocess to execute the JSON instructions, waits
      for the process to complete and dispatches the next task, until all tasks are
      complete or a critical failure ( exception ) is raised.

      If an exception is raised in any task, all following tasks are not executed by design.

      Example: python run.py [path to recipe file] --force
      Caution: This script does NOT check if the last job finished, potentially causing overruns.
      Notes:
        - To avoid running the entire script when debugging a single task, the command line
          can easily replace "all" with the name of any "task" in the json.  For example
          python all/run.py project/sample/say_hello.json

        - Can be easily replaced with the following to run only the "hello" task:
          python task/hello/run.py project/sample/say_hello.json

        - Or specified further to run only the second hello task:
          python task/hello/run.py project/sample/say_hello.json -i 2

  """))

    parser.add_argument('json', help='path to tasks json file')

    parser.add_argument('--project',
                        '-p',
                        help='cloud id of project, defaults to None',
                        default=None)
    parser.add_argument(
        '--user',
        '-u',
        help=
        'path to user credentials json file, defaults to GOOGLE_APPLICATION_CREDENTIALS',
        default=None)
    parser.add_argument(
        '--service',
        '-s',
        help='path to service credentials json file, defaults None',
        default=None)
    parser.add_argument(
        '--client',
        '-c',
        help='path to client credentials json file, defaults None',
        default=None)

    parser.add_argument('--verbose',
                        '-v',
                        help='print all the steps as they happen.',
                        action='store_true')
    parser.add_argument(
        '--date',
        '-d',
        help=
        'YYYY-MM-DD format date for which these reports are to be run, default will be today.',
        default='TODAY')
    parser.add_argument('--force',
                        '-force',
                        help='execute all scripts once then exit.',
                        action='store_true')

    parser.add_argument(
        '--trace_print',
        '-tp',
        help='Simplified execution trace of the program written to stdout.',
        action='store_true')
    parser.add_argument(
        '--trace_file',
        '-tf',
        help='Simplified execution trace of the program written to file.',
        action='store_true')

    args = parser.parse_args()

    # load json to get each task
    recipe = get_project(args.json)

    # track per task instance count
    instances = {}

    returncode = 0
    for task in recipe['tasks']:
        script, task = next(iter(task.items()))

        # count instance per task
        instances.setdefault(script, 0)
        instances[script] += 1

        # assemble command ( replace command, use all arguments passed, and add instance )
        command = 'python -W ignore %s/starthinker/task/%s/run.py %s -i %d' % (
            UI_ROOT, script, ' '.join(sys.argv[1:]), instances[script])

        # show command so user can run each task
        print(command)

        # execute command if schedule, return code
        if args.force or is_scheduled(recipe, task):
            returncode |= subprocess.Popen(command, shell=True).wait()

        # skip command if not schedule
        else:
            raise SystemExit(
                'Schedule Skipping: run command manually or add --force to run all'
            )

    # Set lowest return code from all tasks
    exit(returncode)
Esempio n. 9
0
###########################################################################
"""Evaluate the validity of a json file. Helps in debugging recipes.

Print the line and character position of any errors in the given json file.

Arguments

  file - path to JSON file to be evaluated

Example 

  python project/helper.py project/sample.json

"""

import argparse

from starthinker.util.project import get_project

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('file', help='A JSON file.')
    args = parser.parse_args()

    try:
        project = get_project(args.file, debug=True)
        print 'JSON OK:', args.file
    except Exception, e:
        print 'JSON ERROR:', args.file, str(e)
Esempio n. 10
0
"""Evaluate the validity of a json file. Helps in debugging recipes.

Print the line and character position of any errors in the given json file.

Arguments

  file - path to JSON file to be evaluated

Example 

  python project/helper.py project/sample.json

"""

import argparse
from json import JSONDecodeError

from starthinker.util.project import get_project

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('file', help='A JSON file.')
    args = parser.parse_args()

    try:
        project = get_project(args.file)
        print('JSON OK:', args.file)
    except JSONDecodeError as e:
        print(str(e))
Esempio n. 11
0
 def __init__(self, _recipe_path):
     self.recipe_path = _recipe_path
     self.recipe = get_project(_recipe_path)
     self.dag = None
Esempio n. 12
0
                        help='print all the steps as they happen.',
                        action='store_true')
    parser.add_argument(
        '--date',
        '-d',
        help=
        'YYYY-MM-DD format date for which these reports are to be run, default will be today.',
        default='TODAY')
    parser.add_argument('--force',
                        '-force',
                        help='execute all scripts once then exit.',
                        action='store_true')
    args = parser.parse_args()

    # load json to get each task
    project = get_project(args.json)
    # track per task instance count
    instances = {}
    # return code changes if a task fails
    return_code = 0

    for task in project['tasks']:
        script, task = task.items()[0]

        # count instance per task
        instances.setdefault(script, 0)
        instances[script] += 1

        # assemble command ( replace command, use all arguments passed, and add instance )
        command = 'python %stask/%s/run.py %s -i %d' % (
            EXECUTE_PATH, script, ' '.join(sys.argv[1:]), instances[script])
Esempio n. 13
0
                        action='store_true')
    parser.add_argument('--force',
                        '-f',
                        help='execute all scripts once then exit.',
                        action='store_true')
    #parser.add_argument('--remote', '-r', help='execute the scripts remotely, requires pub/sub config.', action='store_true')

    args = parser.parse_args()

    try:

        while True:
            for filepath in glob('%s*.json' % args.path):
                if args.verbose: print 'PROJECT:', filepath

                project = get_project(filepath)

                if args.force or is_scheduled(project):

                    command = 'python task/%s/run.py %s %s' % (
                        script, filepath, ' '.join(sys.argv[2:]))

                    if args.verbose: print 'COMMAND:', command

                    subprocess.Popen(command, shell=True, cwd=EXECUTE_PATH)

            if args.force: break
            if args.verbose: print 'SLEEP:', ONE_HOUR_AND_ONE_SECOND
            sleep(ONE_HOUR_AND_ONE_SECOND)

    except KeyboardInterrupt:
Esempio n. 14
0
Print the line and character position of any errors in the given json file.

Arguments

  file - path to JSON file to be evaluated

Example 

  python project/helper.py project/sample.json

"""


import argparse

from starthinker.util.project import get_project


if __name__ == "__main__":

  parser = argparse.ArgumentParser()
  parser.add_argument('file', help='A JSON file.')
  parser.add_argument('--debug', '-d', help='Debug mode, do not scrub newlines.', action='store_true')
  args = parser.parse_args()

  try:
    project = get_project(args.file, debug=args.debug)
    print('JSON OK:', args.file)
  except Exception as e:
    print('JSON ERROR:', args.file, str(e))
Esempio n. 15
0
def main():

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description=textwrap.dedent("""\
    Evaluate the validity of a json file. Helps in debugging recipes.
    Print the line and character position of any errors in the given json file.

    Also provide ability to replace values in a recipe with fields and values.

    Examples:
      - python helper.py scripts/say_hello.json
      - python helper.py scripts/dv360_targeting_audit.json -target 'DV_Targeting_Audit.' -value '{dataset}.'
      - python helper.py scripts/dv360_targeting_audit.json -target '"service"' -field "auth_bigquery" -order 3 -description "BigQuery read/ write credentials." -default "service" -kind "authentication"
  """))

    parser.add_argument('file', help='A JSON file.')
    parser.add_argument('-target', help='The string value to replace.')

    parser.add_argument('-delete', action='store_false')

    parser.add_argument('-value', help='The value to insert.', default='')

    parser.add_argument('-field', help='The field name to insert.')
    parser.add_argument('-kind',
                        help='The field kind to insert.',
                        default='string')
    parser.add_argument('-prefix',
                        help='The field prefix to insert.',
                        default='')
    parser.add_argument('-default',
                        help='The field default to insert.',
                        default='')
    parser.add_argument('-description',
                        help='The field description to insert.',
                        default='')
    parser.add_argument('-order',
                        help='The field order to insert.',
                        type=int,
                        default=1)
    parser.add_argument('-choices',
                        help='The field choices to insert.',
                        default='')

    args = parser.parse_args()

    try:
        get_project(args.file)
        print('JSON OK:', args.file)
    except JSONDecodeError as e:
        print('JSON ERROR:', str(e))
        exit

    if args.target:
        with open(args.file) as recipe_file:
            recipe = recipe_file.read()

        value = args.value

        if args.field:
            value = '{"field":{ "name":"%s", "kind":"%s", "order":%d, "default":"%s"' % (
                args.field, args.kind, args.order, args.default)
            if args.prefix:
                value += ', "prefix":"%s"' % args.prefix

            if args.choices:
                value += ', "choices":[%s]' % args.coices

            value += ', "description":"%s" }}' % args.description

        elif args.delete:
            value = ''

        print('REPLACE:', args.target)
        print('WITH:', value)

        recipe = recipe.replace(args.target, value)

        try:
            get_project(None, recipe)
            print('NEW JSON OK:', args.file)
        except JSONDecodeError as e:
            print('NEW JSON ERROR:', str(e))
            exit

        if str(input('Commit change (y/n): ')).lower().strip() == 'y':
            with open(args.file, "w") as f:
                f.write(recipe)
Esempio n. 16
0
from django.conf import settings

from starthinker.config import EXECUTE_PATH
from starthinker.script.parse import json_set_fields, json_set_instructions, json_set_description
from starthinker.util.project import get_project

RE_SCRIPT = re.compile(r'^script_.*\.json$')

# cache scripts and authors in memory
AUTHORS = {}
SCRIPTS = {}
for root, dirs, files in os.walk(EXECUTE_PATH):
    for filename in files:
        if RE_SCRIPT.match(filename):
            try:
                script = get_project(root + '/' + filename)
                if not 'script' in script: continue
                SCRIPTS[filename.replace('script_', '').replace('.json',
                                                                '')] = script
                # authors ( for solution lookup )
                container = root.rsplit('/', 1)[-1]
                AUTHORS[container] = AUTHORS.get(container, set()) | set(
                    script['script'].get('authors', []))
                print 'OK', filename
            except Exception, e:
                print 'ERROR:', filename, str(e)


class Script:
    @staticmethod
    def get_scripts(account_email=None):
Esempio n. 17
0
def load_tests():
    for root, dirs, files in os.walk(TEST_DIRECTORY):
        for filename in files:
            if filename.endswith('.json'):
                print('LOADING', filename)
                yield filename, get_project(TEST_DIRECTORY + filename)
Esempio n. 18
0
    parser.add_argument(
        '--trace_print',
        '-tp',
        help='Simplified execution trace of the program written to stdout.',
        action='store_true')
    parser.add_argument(
        '--trace_file',
        '-tf',
        help='Simplified execution trace of the program written to file.',
        action='store_true')

    args = parser.parse_args()

    # load json to get each task
    recipe = get_project(args.json)

    # track per task instance count
    instances = {}

    returncode = 0
    for task in recipe['tasks']:
        script, task = next(iter(task.items()))

        # count instance per task
        instances.setdefault(script, 0)
        instances[script] += 1

        # assemble command ( replace command, use all arguments passed, and add instance )
        command = 'python -W ignore %s/starthinker/task/%s/run.py %s -i %d' % (
            UI_ROOT, script, ' '.join(sys.argv[1:]), instances[script])