Beispiel #1
0
def set_default(name=None, index=None):
    """
    set the default configuration by name

    """
    default_was_set = False
    count = 1

    for configuration in _CONFIG.sections():
        if index != None:
            if count == index:
                _CONFIG.set(configuration, 'default', True)
                default_was_set = True
            else:
                _CONFIG.remove_option(configuration, 'default')

        if name != None:
            if configuration == name:
                _CONFIG.set(configuration, 'default', True)
                default_was_set = True
            else:
                _CONFIG.remove_option(configuration, 'default')

        count += 1

    if not default_was_set:
        raise JutException('Unable to find %s configuration' % name)

    with open(_CONFIG_FILEPATH, 'w') as configfile:
        _CONFIG.write(configfile)

    info('Configuration updated at %s' % _JUT_HOME)
Beispiel #2
0
def show():
    """
    print the available configurations directly to stdout

    """
    if not is_configured():
        raise JutException('No configurations available, please run: `jut config add`')

    info('Available jut configurations:')
    index = 0
    for configuration in _CONFIG.sections():
        username = _CONFIG.get(configuration, 'username')
        app_url = _CONFIG.get(configuration, 'app_url')

        if app_url != defaults.APP_URL:
            if _CONFIG.has_option(configuration, 'default'):
                info(' %d: %s@%s (default)', index + 1, username, app_url)
            else:
                info(' %d: %s@%s', index + 1, username, app_url)

        else:
            if _CONFIG.has_option(configuration, 'default'):
                info(' %d: %s (default)' % (index + 1, username))
            else:
                info(' %d: %s' % (index + 1, username))

        index += 1
Beispiel #3
0
def list(options):
    """
    list programs that belong to the authenticated user

    """
    configuration = config.get_default()
    app_url = configuration["app_url"]

    if options.deployment != None:
        deployment_name = options.deployment
    else:
        deployment_name = configuration["deployment_name"]

    client_id = configuration["client_id"]
    client_secret = configuration["client_secret"]

    token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url)

    if options.all == True:
        account_id = None

    else:
        account_id = accounts.get_logged_in_account_id(token_manager=token_manager, app_url=app_url)

    programs_details = programs.get_programs(
        deployment_name, token_manager=token_manager, created_by=account_id, app_url=app_url
    )

    account_ids = set()
    for program in programs_details:
        account_ids.add(program["createdBy"])

    accounts_details = accounts.get_accounts(account_ids, token_manager=token_manager, app_url=app_url)

    account_lookup = {}
    for account in accounts_details["accounts"]:
        account_lookup[account["id"]] = account

    headers = ["Name", "Last Saved", "Created By"]
    table = []

    for program in programs_details:
        username = account_lookup[program["createdBy"]]["username"]
        program_name = program["name"]
        last_edited = program["lastEdited"]
        table.append([program_name, last_edited, username])

    if options.format == "table":
        info(tabulate.tabulate(table, headers, tablefmt="orgtbl"))

    elif options.format == "text":
        info(tabulate.tabulate(table, headers, tablefmt="orgtbl", stralign="center"))

    else:
        raise JutException('Unsupported format "%s"' % options.format)
Beispiel #4
0
    def point(self, point):
        line = []

        if 'time' in point:
            timestamp = point['time']
            del point['time']
            line.append(timestamp)

        keys = sorted(point.keys())
        line += [str(point[key]) for key in keys]
        info(' '.join(line))
Beispiel #5
0
    def expect_status(self, expected_status):
        """
        expect the provided status exit code otherwise output the full stdout
        and stderr output at that specific point in time

        """
        status = self.wait()

        if status != expected_status:
            info(self.read_output())
            error(self.read_error())
            raise Exception('Expected status %s, got %s' % (expected_status, status))
Beispiel #6
0
def add(name, **kwargs):
    """
    add a new configuration with the name specified and all of the keywords
    as attributes of that configuration.

    """
    _CONFIG.add_section(name)

    for (key, value) in kwargs.items():
        _CONFIG.set(name, key, value)

    with open(_CONFIG_FILEPATH, 'w') as configfile:
        _CONFIG.write(configfile)

    info('Configuration updated at %s' % _JUT_HOME)
Beispiel #7
0
    def point(self, point):
        line = []

        if 'time' in point:
            keys = sorted(point.keys())
            keys.remove('time')
            keys.insert(0, 'time')

        else:
            keys = sorted(point.keys())

        if self.current_headers != keys:
            info('#%s' % ','.join(keys))
            self.current_headers = keys

        line += [str(point[key]) for key in keys]
        info(','.join(line))
Beispiel #8
0
def post(json_data,
         url,
         dry_run=False):
    """
    POST json data to the url provided and verify the requests was successful

    """

    if dry_run:
        info('POST: %s' % json.dumps(json_data, indent=4))
    else:
        response = SESSION.post(url,
                                data=json.dumps(json_data),
                                headers={'content-type': 'application/json'})

        if response.status_code != 200:
            raise Exception("Failed to import %s with %s: %s" %
                            (json_data, response.status_code, response.text))
Beispiel #9
0
def upload_file(options):
    if not sys.stdin.isatty():
        json_file = sys.stdin

    else:
        json_file = open(options.source, 'r')

    url = options.url

    if url == None:
        configuration = config.get_default()
        app_url = configuration['app_url']

        if options.deployment != None:
            deployment_name = options.deployment
        else:
            deployment_name = configuration['deployment_name']

        client_id = configuration['client_id']
        client_secret = configuration['client_secret']

        token_manager = auth.TokenManager(client_id=client_id,
                                          client_secret=client_secret,
                                          app_url=app_url)

        url = integrations.get_webhook_url(deployment_name,
                                           space=options.space,
                                           token_manager=token_manager,
                                           app_url=app_url)

    info('Pushing to %s' % url)
    push_json_file(json_file,
                   url,
                   dry_run=options.dry_run,
                   batch_size=options.batch_size,
                   anonymize_fields=options.anonymize_fields,
                   remove_fields=options.remove_fields,
                   rename_fields=options.rename_fields)
Beispiel #10
0
    def expect_output(self, message):
        """
        expect the stdout contains the following message in its output before
        proceeding

        """
        # use select to timeout when there is no output
        read_ready, _, _ = select.select([self.process.stdout.fileno()], [], [], 5)

        if read_ready:
            length = len(message)
            line = self.process.stdout.read(length)

            if message == line:
                return

            info(self.read_output())
            error(self.read_error())
            raise Exception('Expected "%s" got "%s"' % (message, line))
        else:
            info(self.read_output())
            error(self.read_error())
            raise Exception('Expected "%s" got nothing' % message)
Beispiel #11
0
def push(options):
    configuration = config.get_default()
    app_url = configuration["app_url"]

    if options.deployment != None:
        deployment_name = options.deployment
    else:
        deployment_name = configuration["deployment_name"]

    client_id = configuration["client_id"]
    client_secret = configuration["client_secret"]

    token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url)

    if not os.path.exists(options.source):
        raise JutException('Source "%s" does not exists.')

    filenames = []
    if os.path.isdir(options.source):
        for filename in os.listdir(options.source):
            if filename.endswith(".juttle"):
                filenames.append(filename)

    else:
        filenames.append(options.source)

    decision = None
    for filename in filenames:
        filepath = os.path.join(options.source, filename)
        program_name = urllib.unquote_plus(os.path.basename(filepath).replace(r".juttle", ""))
        info('Found program "%s"' % program_name)

        with codecs.open(filepath, "r", encoding="UTF-8") as program_file:
            program_code = program_file.read()

        local_last_edited = int(os.stat(filepath).st_mtime)

        if programs.program_exists(program_name, deployment_name, token_manager=token_manager, app_url=app_url):

            # one last safety to check if the modification time of
            # the file still matches the lastEdited of the existing
            # copy on Jut otherwise we prompt the user for confirmation
            program = programs.get_program(program_name, deployment_name, token_manager=token_manager, app_url=app_url)

            remote_last_edited = dates.iso8601_to_epoch(program["lastEdited"])

            if local_last_edited != remote_last_edited and decision != "A":
                info('Juttle changed since last pull for "%s"' % program_name)
                decision = console.prompt(
                    "Would you like to " "(O - Override," " S - Skip," " R - Review Changes," " A - override All)?"
                )

                if decision == "R":
                    info("Following is what would change if we overrode using your copy:")
                    info("*" * 80)
                    for line in difflib.ndiff(program["code"].split("\n"), program_code.split("\n")):
                        info(line)
                    info("*" * 80)
                    decision = console.prompt("Would you like to " "(O - Override," " S - Skip)?")

                if decision == "S":
                    # jump to the next file
                    continue

                elif decision == "O":
                    pass

                elif decision == "A":
                    pass

                else:
                    raise JutException('Unexpected option "%s"' % decision)

            last_edited_iso = dates.epoch_to_iso8601(local_last_edited)
            programs.update_program(
                program_name,
                program_code,
                deployment_name,
                last_edited=last_edited_iso,
                token_manager=token_manager,
                app_url=app_url,
            )
            os.utime(filepath, (local_last_edited, local_last_edited))

        else:
            last_edited_iso = dates.epoch_to_iso8601(local_last_edited)
            programs.save_program(
                program_name,
                program_code,
                deployment_name,
                last_edited=last_edited_iso,
                token_manager=token_manager,
                app_url=app_url,
            )
            os.utime(filepath, (local_last_edited, local_last_edited))
Beispiel #12
0
def _print_jobs(jobs, token_manager, app_url, options):
    """
    internal method to print the provided jobs array in a nice tabular
    format

    """
    accountids = set()
    for job in jobs:
        if job['user'] != 'jut.internal.user':
            accountids.add(job['user'])

    account_lookup = {
        'jut.internal.user': {
            'username': '******'
        }
    }

    if accountids:
        accounts_details = accounts.get_accounts(accountids,
                                                 token_manager=token_manager,
                                                 app_url=app_url)

        for account in accounts_details['accounts']:
            account_lookup[account['id']] = account

    if options.format == 'text':
        labels = OrderedDict()
        labels['id'] = 'Job ID'
        labels['alias'] = 'Juttle Name'
        labels['username'] = '******'
        labels['_start_time'] = 'Start Date'
        labels['persistent'] = 'Persistent'

        max_lengths = {
            'id': 0,
            'alias': 0,
            'username': 0,
            '_start_time': 0,
            'persistent': 0,
        }

        for key in max_lengths.keys():
            max_lengths[key] = len(labels[key]) + 1

        # retrieve username and fix up persistent marker
        for job in jobs:
            job['username'] = account_lookup[job['user']]['username']
            job['persistent'] = 'YES' if job['timeout'] == 0 else 'NO'

        # calculate max length of each column
        for job in jobs:
            for key in labels.keys():
                if max_lengths[key] < len(job[key]):
                    max_lengths[key] = len(job[key]) + 1

        # print labels
        header = ''
        for key in labels.keys():
            header += (labels[key] + ' ' * (max_lengths[key] - len(labels[key])))

        info(header)

        for job in jobs:
            line = ''

            for key in labels.keys():
                line += (job[key] + ' ' * (max_lengths[key] - len(job[key])))

            info(line)

    elif options.format == 'table':
        headers = ['Job ID', 'Juttle Name', 'Owner', 'Start Date', 'Persistent']

        table = []
        for job in jobs:
            owner = account_lookup[job['user']]['username']
            persistent = 'YES' if job['timeout'] == 0 else 'NO'
            name = ''

            if 'alias' in job:
                name = job['alias']

            table.append([job['id'],
                          name,
                          owner,
                          job['_start_time'],
                          persistent])

        info(tabulate.tabulate(table, headers, tablefmt="orgtbl"))

    else:
        raise JutException('Unsupported output format "%s"' %
                           options.format)
Beispiel #13
0
def connect(options):
    options.persist = False

    if not config.is_configured():
        configs.add_configuration(options)

    configuration = config.get_default()
    app_url = configuration['app_url']

    if options.deployment != None:
        deployment_name = options.deployment
    else:
        deployment_name = configuration['deployment_name']

    client_id = configuration['client_id']
    client_secret = configuration['client_secret']

    token_manager = auth.TokenManager(client_id=client_id,
                                      client_secret=client_secret,
                                      app_url=app_url)

    total_points = 0

    def show_progress():
        if options.show_progress:
            error('streamed %s points', total_points, end='\r')

    def show_error_or_warning(data):
        """
        handle error and warning reporting

        """
        if 'error' in data:
            prefix = 'Error'

        elif 'warning' in data:
            prefix = 'Warning'

        else:
            raise Exception('Unexpected error/warning received %s' % data)

        message = None
        location = None

        if 'context' in data:
            message = data['context']['message']

            # not all errors or warnings have location information
            if 'location' in data['context']['info']:
                location = data['context']['info']['location']
                line = location['start']['line']
                column = location['start']['column']

        else:
            message = '%s: %s' % (prefix, message)

        if location != None:
            error('%s line %s, column %s of %s: %s' %
                  (prefix, line, column, location['filename'], message))

        else:
            error(message)

    if options.format == 'json':
        formatter = JSONFormatter(options)

    elif options.format == 'text':
        formatter = TextFormatter(options)

    elif options.format == 'csv':
        formatter = CSVFormatter(options)

    else:
        raise JutException('Unsupported output format "%s"' %
                           options.format)

    job_id = options.job_id

    done = False
    with_errors = False
    max_retries = options.retry
    retry_delay = options.retry_delay
    retry = 0

    while not done:
        try:
            if not options.persist:
                formatter.start()

            for data in data_engine.connect_job(job_id,
                                                deployment_name,
                                                token_manager=token_manager,
                                                app_url=app_url):
                show_progress()

                if 'job' in data:
                    # job details
                    if options.persist:
                        # lets print the job id
                        info(data['job']['id'])

                if 'points' in data:
                    points = data['points']
                    for point in points:
                        formatter.point(point)

                    total_points += len(points)

                elif 'error' in data:
                    show_error_or_warning(data)
                    with_errors = True

                elif 'warning' in data:
                    show_error_or_warning(data)

            done = True

        except JutException:
            retry += 1

            if max_retries != -1 and retry > max_retries:
                raise

            time.sleep(retry_delay)

        finally:
            if options.show_progress:
                # one enter to retain the last value of progress output
                info('')

            if not options.persist:
                formatter.stop()

            if with_errors:
                raise JutException('Error while running juttle')
Beispiel #14
0
 def stop(self):
     if not self.options.persist:
         info('\n]')
Beispiel #15
0
 def start(self):
     if not self.options.persist:
         info('[')
Beispiel #16
0
def init():
    """
    initialize the testing configuration

    """

    # set PYTHONPATH to the cwd directory because we want to test this
    # source and not any installed version of the jut tools
    os.environ.setdefault('PYTHONPATH', '.')

    if os.environ.get('JUT_USER') == None or \
    os.environ.get('JUT_PASS') == None:
        info('')
        info('You need to set JUT_USER and JUT_PASS to a valid jut admin user ')
        info('like so:')
        info('')
        info(' JUT_USER=username JUT_PASS=password python setup.py test')
        info('')
        sys.exit(1)

    info('')
    info('*'*80)
    info('During testing we will create a few test accounts and spaces to ')
    info('verify different features in the jut-tools. We will do our best to ')
    info('clean those up but if you see anything starting with jut-tools you ')
    info('will now that was left behind by the unit tests here')
    info('*'*80)
    info('')

    # we set the HOME_OVERRIDE for testing
    os.environ.setdefault('HOME_OVERRIDE', tempfile.mkdtemp())

    # configure the default account
    setup_command = 'python jut/cli.py config add -u %s -p %s -d' % \
                    (os.environ.get('JUT_USER'), os.environ.get('JUT_PASS'))

    if os.environ.get('JUT_APP_URL') != None:
        setup_command += ' -a "%s"' % os.environ.get('JUT_APP_URL')

    if os.system(setup_command) != 0:
        error('Failed to create testing configuration')
Beispiel #17
0
def pull(options):
    """
    pull all remote programs to a local directory

    """
    configuration = config.get_default()
    app_url = configuration["app_url"]

    if options.deployment != None:
        deployment_name = options.deployment
    else:
        deployment_name = configuration["deployment_name"]

    client_id = configuration["client_id"]
    client_secret = configuration["client_secret"]

    token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url)

    if options.all == True:
        account_id = None

    else:
        account_id = accounts.get_logged_in_account_id(token_manager=token_manager, app_url=app_url)

    programs_details = programs.get_programs(
        deployment_name, token_manager=token_manager, created_by=account_id, app_url=app_url
    )

    if not os.path.exists(options.directory):
        os.mkdir(options.directory)

    account_ids = set()
    for program in programs_details:
        account_ids.add(program["createdBy"])

    accounts_details = accounts.get_accounts(account_ids, token_manager=token_manager, app_url=app_url)

    account_lookup = {}
    for account in accounts_details["accounts"]:
        account_lookup[account["id"]] = account

    decision = None
    for program in programs_details:
        program_name = program["name"]
        juttle_filename = "%s.juttle" % escape_filename(program_name)

        if options.per_user_directory:
            username = account_lookup[program["createdBy"]]["username"]
            userdir = os.path.join(options.directory, username)

            if not os.path.exists(userdir):
                os.mkdir(userdir)

            juttle_filepath = os.path.join(userdir, juttle_filename)

        else:
            juttle_filepath = os.path.join(options.directory, juttle_filename)

        if os.path.exists(juttle_filepath) and decision != "A":
            program_code = None
            with codecs.open(juttle_filepath, "r", encoding="UTF-8") as program_file:
                program_code = program_file.read()

            local_last_edited = int(os.stat(juttle_filepath).st_mtime)
            remote_last_edited = dates.iso8601_to_epoch(program["lastEdited"])

            if local_last_edited != remote_last_edited:
                info('Juttle changed since last pull for "%s"' % program_name)
                decision = console.prompt(
                    "Would you like to " "(O - Override," " S - Skip," " R - Review Changes," " A - override All)?"
                )

                if decision == "R":
                    info("Following is what would change if we overrode using your copy:")
                    info("*" * 80)
                    for line in difflib.ndiff(program["code"].split("\n"), program_code.split("\n")):
                        info(line)
                    info("*" * 80)
                    decision = console.prompt("Would you like to " "(O - Override," " S - Skip)?")

                if decision == "S":
                    # jump to the next file
                    continue

                elif decision == "O":
                    pass

                elif decision == "A":
                    pass

                else:
                    raise JutException('Unexpected option "%s"' % decision)

        info('importing program "%s" to %s' % (program["name"], juttle_filepath))
        with codecs.open(juttle_filepath, "w", encoding="UTF-8") as program_file:
            program_file.write(program["code"])

        # update creation time to match the lastEdited field
        epoch = dates.iso8601_to_epoch(program["lastEdited"])
        os.utime(juttle_filepath, (epoch, epoch))
Beispiel #18
0
 def point(self, point):
     if self.previous_point:
         info(',')
     info(json.dumps(point, indent=2), end='')
     self.previous_point = True