def set_default(name=None, index=None):
"""
set the default configuration by name
"""
default_was_set = False
count = 1
for configuration in _CONFIG.sections():
if index != None:
if count == index:
_CONFIG.set(configuration, 'default', True)
default_was_set = True
else:
_CONFIG.remove_option(configuration, 'default')
if name != None:
if configuration == name:
_CONFIG.set(configuration, 'default', True)
default_was_set = True
else:
_CONFIG.remove_option(configuration, 'default')
count += 1
if not default_was_set:
raise JutException('Unable to find %s configuration' % name)
with open(_CONFIG_FILEPATH, 'w') as configfile:
_CONFIG.write(configfile)
info('Configuration updated at %s' % _JUT_HOME)
def show():
"""
print the available configurations directly to stdout
"""
if not is_configured():
raise JutException('No configurations available, please run: `jut config add`')
info('Available jut configurations:')
index = 0
for configuration in _CONFIG.sections():
username = _CONFIG.get(configuration, 'username')
app_url = _CONFIG.get(configuration, 'app_url')
if app_url != defaults.APP_URL:
if _CONFIG.has_option(configuration, 'default'):
info(' %d: %s@%s (default)', index + 1, username, app_url)
else:
info(' %d: %s@%s', index + 1, username, app_url)
else:
if _CONFIG.has_option(configuration, 'default'):
info(' %d: %s (default)' % (index + 1, username))
else:
info(' %d: %s' % (index + 1, username))
index += 1
def list(options):
"""
list programs that belong to the authenticated user
"""
configuration = config.get_default()
app_url = configuration["app_url"]
if options.deployment != None:
deployment_name = options.deployment
else:
deployment_name = configuration["deployment_name"]
client_id = configuration["client_id"]
client_secret = configuration["client_secret"]
token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url)
if options.all == True:
account_id = None
else:
account_id = accounts.get_logged_in_account_id(token_manager=token_manager, app_url=app_url)
programs_details = programs.get_programs(
deployment_name, token_manager=token_manager, created_by=account_id, app_url=app_url
)
account_ids = set()
for program in programs_details:
account_ids.add(program["createdBy"])
accounts_details = accounts.get_accounts(account_ids, token_manager=token_manager, app_url=app_url)
account_lookup = {}
for account in accounts_details["accounts"]:
account_lookup[account["id"]] = account
headers = ["Name", "Last Saved", "Created By"]
table = []
for program in programs_details:
username = account_lookup[program["createdBy"]]["username"]
program_name = program["name"]
last_edited = program["lastEdited"]
table.append([program_name, last_edited, username])
if options.format == "table":
info(tabulate.tabulate(table, headers, tablefmt="orgtbl"))
elif options.format == "text":
info(tabulate.tabulate(table, headers, tablefmt="orgtbl", stralign="center"))
else:
raise JutException('Unsupported format "%s"' % options.format)
def point(self, point):
line = []
if 'time' in point:
timestamp = point['time']
del point['time']
line.append(timestamp)
keys = sorted(point.keys())
line += [str(point[key]) for key in keys]
info(' '.join(line))
def expect_status(self, expected_status):
"""
expect the provided status exit code otherwise output the full stdout
and stderr output at that specific point in time
"""
status = self.wait()
if status != expected_status:
info(self.read_output())
error(self.read_error())
raise Exception('Expected status %s, got %s' % (expected_status, status))
def add(name, **kwargs):
"""
add a new configuration with the name specified and all of the keywords
as attributes of that configuration.
"""
_CONFIG.add_section(name)
for (key, value) in kwargs.items():
_CONFIG.set(name, key, value)
with open(_CONFIG_FILEPATH, 'w') as configfile:
_CONFIG.write(configfile)
info('Configuration updated at %s' % _JUT_HOME)
def point(self, point):
line = []
if 'time' in point:
keys = sorted(point.keys())
keys.remove('time')
keys.insert(0, 'time')
else:
keys = sorted(point.keys())
if self.current_headers != keys:
info('#%s' % ','.join(keys))
self.current_headers = keys
line += [str(point[key]) for key in keys]
info(','.join(line))
def post(json_data,
url,
dry_run=False):
"""
POST json data to the url provided and verify the requests was successful
"""
if dry_run:
info('POST: %s' % json.dumps(json_data, indent=4))
else:
response = SESSION.post(url,
data=json.dumps(json_data),
headers={'content-type': 'application/json'})
if response.status_code != 200:
raise Exception("Failed to import %s with %s: %s" %
(json_data, response.status_code, response.text))
def upload_file(options):
if not sys.stdin.isatty():
json_file = sys.stdin
else:
json_file = open(options.source, 'r')
url = options.url
if url == None:
configuration = config.get_default()
app_url = configuration['app_url']
if options.deployment != None:
deployment_name = options.deployment
else:
deployment_name = configuration['deployment_name']
client_id = configuration['client_id']
client_secret = configuration['client_secret']
token_manager = auth.TokenManager(client_id=client_id,
client_secret=client_secret,
app_url=app_url)
url = integrations.get_webhook_url(deployment_name,
space=options.space,
token_manager=token_manager,
app_url=app_url)
info('Pushing to %s' % url)
push_json_file(json_file,
url,
dry_run=options.dry_run,
batch_size=options.batch_size,
anonymize_fields=options.anonymize_fields,
remove_fields=options.remove_fields,
rename_fields=options.rename_fields)
def expect_output(self, message):
"""
expect the stdout contains the following message in its output before
proceeding
"""
# use select to timeout when there is no output
read_ready, _, _ = select.select([self.process.stdout.fileno()], [], [], 5)
if read_ready:
length = len(message)
line = self.process.stdout.read(length)
if message == line:
return
info(self.read_output())
error(self.read_error())
raise Exception('Expected "%s" got "%s"' % (message, line))
else:
info(self.read_output())
error(self.read_error())
raise Exception('Expected "%s" got nothing' % message)
def push(options):
configuration = config.get_default()
app_url = configuration["app_url"]
if options.deployment != None:
deployment_name = options.deployment
else:
deployment_name = configuration["deployment_name"]
client_id = configuration["client_id"]
client_secret = configuration["client_secret"]
token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url)
if not os.path.exists(options.source):
raise JutException('Source "%s" does not exists.')
filenames = []
if os.path.isdir(options.source):
for filename in os.listdir(options.source):
if filename.endswith(".juttle"):
filenames.append(filename)
else:
filenames.append(options.source)
decision = None
for filename in filenames:
filepath = os.path.join(options.source, filename)
program_name = urllib.unquote_plus(os.path.basename(filepath).replace(r".juttle", ""))
info('Found program "%s"' % program_name)
with codecs.open(filepath, "r", encoding="UTF-8") as program_file:
program_code = program_file.read()
local_last_edited = int(os.stat(filepath).st_mtime)
if programs.program_exists(program_name, deployment_name, token_manager=token_manager, app_url=app_url):
# one last safety to check if the modification time of
# the file still matches the lastEdited of the existing
# copy on Jut otherwise we prompt the user for confirmation
program = programs.get_program(program_name, deployment_name, token_manager=token_manager, app_url=app_url)
remote_last_edited = dates.iso8601_to_epoch(program["lastEdited"])
if local_last_edited != remote_last_edited and decision != "A":
info('Juttle changed since last pull for "%s"' % program_name)
decision = console.prompt(
"Would you like to " "(O - Override," " S - Skip," " R - Review Changes," " A - override All)?"
)
if decision == "R":
info("Following is what would change if we overrode using your copy:")
info("*" * 80)
for line in difflib.ndiff(program["code"].split("\n"), program_code.split("\n")):
info(line)
info("*" * 80)
decision = console.prompt("Would you like to " "(O - Override," " S - Skip)?")
if decision == "S":
# jump to the next file
continue
elif decision == "O":
pass
elif decision == "A":
pass
else:
raise JutException('Unexpected option "%s"' % decision)
last_edited_iso = dates.epoch_to_iso8601(local_last_edited)
programs.update_program(
program_name,
program_code,
deployment_name,
last_edited=last_edited_iso,
token_manager=token_manager,
app_url=app_url,
)
os.utime(filepath, (local_last_edited, local_last_edited))
else:
last_edited_iso = dates.epoch_to_iso8601(local_last_edited)
programs.save_program(
program_name,
program_code,
deployment_name,
last_edited=last_edited_iso,
token_manager=token_manager,
app_url=app_url,
)
os.utime(filepath, (local_last_edited, local_last_edited))
def _print_jobs(jobs, token_manager, app_url, options):
"""
internal method to print the provided jobs array in a nice tabular
format
"""
accountids = set()
for job in jobs:
if job['user'] != 'jut.internal.user':
accountids.add(job['user'])
account_lookup = {
'jut.internal.user': {
'username': '******'
}
}
if accountids:
accounts_details = accounts.get_accounts(accountids,
token_manager=token_manager,
app_url=app_url)
for account in accounts_details['accounts']:
account_lookup[account['id']] = account
if options.format == 'text':
labels = OrderedDict()
labels['id'] = 'Job ID'
labels['alias'] = 'Juttle Name'
labels['username'] = '******'
labels['_start_time'] = 'Start Date'
labels['persistent'] = 'Persistent'
max_lengths = {
'id': 0,
'alias': 0,
'username': 0,
'_start_time': 0,
'persistent': 0,
}
for key in max_lengths.keys():
max_lengths[key] = len(labels[key]) + 1
# retrieve username and fix up persistent marker
for job in jobs:
job['username'] = account_lookup[job['user']]['username']
job['persistent'] = 'YES' if job['timeout'] == 0 else 'NO'
# calculate max length of each column
for job in jobs:
for key in labels.keys():
if max_lengths[key] < len(job[key]):
max_lengths[key] = len(job[key]) + 1
# print labels
header = ''
for key in labels.keys():
header += (labels[key] + ' ' * (max_lengths[key] - len(labels[key])))
info(header)
for job in jobs:
line = ''
for key in labels.keys():
line += (job[key] + ' ' * (max_lengths[key] - len(job[key])))
info(line)
elif options.format == 'table':
headers = ['Job ID', 'Juttle Name', 'Owner', 'Start Date', 'Persistent']
table = []
for job in jobs:
owner = account_lookup[job['user']]['username']
persistent = 'YES' if job['timeout'] == 0 else 'NO'
name = ''
if 'alias' in job:
name = job['alias']
table.append([job['id'],
name,
owner,
job['_start_time'],
persistent])
info(tabulate.tabulate(table, headers, tablefmt="orgtbl"))
else:
raise JutException('Unsupported output format "%s"' %
options.format)
def connect(options):
options.persist = False
if not config.is_configured():
configs.add_configuration(options)
configuration = config.get_default()
app_url = configuration['app_url']
if options.deployment != None:
deployment_name = options.deployment
else:
deployment_name = configuration['deployment_name']
client_id = configuration['client_id']
client_secret = configuration['client_secret']
token_manager = auth.TokenManager(client_id=client_id,
client_secret=client_secret,
app_url=app_url)
total_points = 0
def show_progress():
if options.show_progress:
error('streamed %s points', total_points, end='\r')
def show_error_or_warning(data):
"""
handle error and warning reporting
"""
if 'error' in data:
prefix = 'Error'
elif 'warning' in data:
prefix = 'Warning'
else:
raise Exception('Unexpected error/warning received %s' % data)
message = None
location = None
if 'context' in data:
message = data['context']['message']
# not all errors or warnings have location information
if 'location' in data['context']['info']:
location = data['context']['info']['location']
line = location['start']['line']
column = location['start']['column']
else:
message = '%s: %s' % (prefix, message)
if location != None:
error('%s line %s, column %s of %s: %s' %
(prefix, line, column, location['filename'], message))
else:
error(message)
if options.format == 'json':
formatter = JSONFormatter(options)
elif options.format == 'text':
formatter = TextFormatter(options)
elif options.format == 'csv':
formatter = CSVFormatter(options)
else:
raise JutException('Unsupported output format "%s"' %
options.format)
job_id = options.job_id
done = False
with_errors = False
max_retries = options.retry
retry_delay = options.retry_delay
retry = 0
while not done:
try:
if not options.persist:
formatter.start()
for data in data_engine.connect_job(job_id,
deployment_name,
token_manager=token_manager,
app_url=app_url):
show_progress()
if 'job' in data:
# job details
if options.persist:
# lets print the job id
info(data['job']['id'])
if 'points' in data:
points = data['points']
for point in points:
formatter.point(point)
total_points += len(points)
elif 'error' in data:
show_error_or_warning(data)
with_errors = True
elif 'warning' in data:
show_error_or_warning(data)
done = True
except JutException:
retry += 1
if max_retries != -1 and retry > max_retries:
raise
time.sleep(retry_delay)
finally:
if options.show_progress:
# one enter to retain the last value of progress output
info('')
if not options.persist:
formatter.stop()
if with_errors:
raise JutException('Error while running juttle')
def stop(self):
if not self.options.persist:
info('\n]')
def start(self):
if not self.options.persist:
info('[')
def init():
"""
initialize the testing configuration
"""
# set PYTHONPATH to the cwd directory because we want to test this
# source and not any installed version of the jut tools
os.environ.setdefault('PYTHONPATH', '.')
if os.environ.get('JUT_USER') == None or \
os.environ.get('JUT_PASS') == None:
info('')
info('You need to set JUT_USER and JUT_PASS to a valid jut admin user ')
info('like so:')
info('')
info(' JUT_USER=username JUT_PASS=password python setup.py test')
info('')
sys.exit(1)
info('')
info('*'*80)
info('During testing we will create a few test accounts and spaces to ')
info('verify different features in the jut-tools. We will do our best to ')
info('clean those up but if you see anything starting with jut-tools you ')
info('will now that was left behind by the unit tests here')
info('*'*80)
info('')
# we set the HOME_OVERRIDE for testing
os.environ.setdefault('HOME_OVERRIDE', tempfile.mkdtemp())
# configure the default account
setup_command = 'python jut/cli.py config add -u %s -p %s -d' % \
(os.environ.get('JUT_USER'), os.environ.get('JUT_PASS'))
if os.environ.get('JUT_APP_URL') != None:
setup_command += ' -a "%s"' % os.environ.get('JUT_APP_URL')
if os.system(setup_command) != 0:
error('Failed to create testing configuration')
def pull(options):
"""
pull all remote programs to a local directory
"""
configuration = config.get_default()
app_url = configuration["app_url"]
if options.deployment != None:
deployment_name = options.deployment
else:
deployment_name = configuration["deployment_name"]
client_id = configuration["client_id"]
client_secret = configuration["client_secret"]
token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url)
if options.all == True:
account_id = None
else:
account_id = accounts.get_logged_in_account_id(token_manager=token_manager, app_url=app_url)
programs_details = programs.get_programs(
deployment_name, token_manager=token_manager, created_by=account_id, app_url=app_url
)
if not os.path.exists(options.directory):
os.mkdir(options.directory)
account_ids = set()
for program in programs_details:
account_ids.add(program["createdBy"])
accounts_details = accounts.get_accounts(account_ids, token_manager=token_manager, app_url=app_url)
account_lookup = {}
for account in accounts_details["accounts"]:
account_lookup[account["id"]] = account
decision = None
for program in programs_details:
program_name = program["name"]
juttle_filename = "%s.juttle" % escape_filename(program_name)
if options.per_user_directory:
username = account_lookup[program["createdBy"]]["username"]
userdir = os.path.join(options.directory, username)
if not os.path.exists(userdir):
os.mkdir(userdir)
juttle_filepath = os.path.join(userdir, juttle_filename)
else:
juttle_filepath = os.path.join(options.directory, juttle_filename)
if os.path.exists(juttle_filepath) and decision != "A":
program_code = None
with codecs.open(juttle_filepath, "r", encoding="UTF-8") as program_file:
program_code = program_file.read()
local_last_edited = int(os.stat(juttle_filepath).st_mtime)
remote_last_edited = dates.iso8601_to_epoch(program["lastEdited"])
if local_last_edited != remote_last_edited:
info('Juttle changed since last pull for "%s"' % program_name)
decision = console.prompt(
"Would you like to " "(O - Override," " S - Skip," " R - Review Changes," " A - override All)?"
)
if decision == "R":
info("Following is what would change if we overrode using your copy:")
info("*" * 80)
for line in difflib.ndiff(program["code"].split("\n"), program_code.split("\n")):
info(line)
info("*" * 80)
decision = console.prompt("Would you like to " "(O - Override," " S - Skip)?")
if decision == "S":
# jump to the next file
continue
elif decision == "O":
pass
elif decision == "A":
pass
else:
raise JutException('Unexpected option "%s"' % decision)
info('importing program "%s" to %s' % (program["name"], juttle_filepath))
with codecs.open(juttle_filepath, "w", encoding="UTF-8") as program_file:
program_file.write(program["code"])
# update creation time to match the lastEdited field
epoch = dates.iso8601_to_epoch(program["lastEdited"])
os.utime(juttle_filepath, (epoch, epoch))
def point(self, point):
if self.previous_point:
info(',')
info(json.dumps(point, indent=2), end='')
self.previous_point = True
