def connect(options):
options.persist = False
if not config.is_configured():
configs.add_configuration(options)
configuration = config.get_default()
app_url = configuration['app_url']
if options.deployment != None:
deployment_name = options.deployment
else:
deployment_name = configuration['deployment_name']
client_id = configuration['client_id']
client_secret = configuration['client_secret']
token_manager = auth.TokenManager(client_id=client_id,
client_secret=client_secret,
app_url=app_url)
total_points = 0
def show_progress():
if options.show_progress:
error('streamed %s points', total_points, end='\r')
def show_error_or_warning(data):
"""
handle error and warning reporting
"""
if 'error' in data:
prefix = 'Error'
elif 'warning' in data:
prefix = 'Warning'
else:
raise Exception('Unexpected error/warning received %s' % data)
message = None
location = None
if 'context' in data:
message = data['context']['message']
# not all errors or warnings have location information
if 'location' in data['context']['info']:
location = data['context']['info']['location']
line = location['start']['line']
column = location['start']['column']
else:
message = '%s: %s' % (prefix, message)
if location != None:
error('%s line %s, column %s of %s: %s' %
(prefix, line, column, location['filename'], message))
else:
error(message)
if options.format == 'json':
formatter = JSONFormatter(options)
elif options.format == 'text':
formatter = TextFormatter(options)
elif options.format == 'csv':
formatter = CSVFormatter(options)
else:
raise JutException('Unsupported output format "%s"' %
options.format)
job_id = options.job_id
done = False
with_errors = False
max_retries = options.retry
retry_delay = options.retry_delay
retry = 0
while not done:
try:
if not options.persist:
formatter.start()
for data in data_engine.connect_job(job_id,
deployment_name,
token_manager=token_manager,
app_url=app_url):
show_progress()
if 'job' in data:
# job details
if options.persist:
# lets print the job id
info(data['job']['id'])
if 'points' in data:
points = data['points']
for point in points:
formatter.point(point)
total_points += len(points)
elif 'error' in data:
show_error_or_warning(data)
with_errors = True
elif 'warning' in data:
show_error_or_warning(data)
done = True
except JutException:
retry += 1
if max_retries != -1 and retry > max_retries:
raise
time.sleep(retry_delay)
finally:
if options.show_progress:
# one enter to retain the last value of progress output
info('')
if not options.persist:
formatter.stop()
if with_errors:
raise JutException('Error while running juttle')
def main():
class JutArgParser(argparse.ArgumentParser):
"""
custom argument parser so we show the full comand line help menu
"""
def error(self, message):
error(message)
self.print_help()
sys.exit(2)
parser = JutArgParser(description='jut - jut command line tools')
commands = parser.add_subparsers(dest='subcommand')
# config commands
config_parser = commands.add_parser('config',
help='configuration management')
config_commands = config_parser.add_subparsers(dest='config_subcommand')
_ = config_commands.add_parser('list',
help='list configurations')
defaults_config = config_commands.add_parser('defaults',
help='change the configuration defaults')
defaults_config.add_argument('-u', '--username',
help='username to use')
defaults_config.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io '
'INTERNAL USE)')
add_config = config_commands.add_parser('add',
help='add another configuration '
'(default when no sub command '
'is provided)')
add_config.add_argument('-u', '--username',
help='username to use')
add_config.add_argument('-p', '--password',
help='password to use')
add_config.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
add_config.add_argument('-d', '--default',
action='store_true',
help='sets this configuration to the default')
add_config.add_argument('-s', '--show-password',
action='store_true',
default=False,
help='shows password as you type it interactively')
rm_config = config_commands.add_parser('rm',
help='remove a configuration')
rm_config.add_argument('-u', '--username',
help='username to use')
rm_config.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
# jobs commands
jobs_parser = commands.add_parser('jobs',
help='jobs management')
jobs_commands = jobs_parser.add_subparsers(dest='jobs_subcommand')
list_jobs = jobs_commands.add_parser('list',
help='list running jobs')
list_jobs.add_argument('-d', '--deployment',
default=None,
help='specify the deployment name')
list_jobs.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
list_jobs.add_argument('-f', '--format',
default='table',
help='available formats are text, table with '
'default: table')
kill_job = jobs_commands.add_parser('kill',
help='kill running job')
kill_job.add_argument('job_id',
help='specify the job_id to kill')
kill_job.add_argument('-d', '--deployment',
default=None,
help='specify the deployment name')
kill_job.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
kill_job.add_argument('-y', '--yes',
action='store_true',
default=False,
help='kill without prompting for confirmation')
connect_job = jobs_commands.add_parser('connect',
help='connect to a persistent job')
connect_job.add_argument('job_id',
help='specify the job_id to connect to')
connect_job.add_argument('-d', '--deployment',
default=None,
help='specify the deployment name')
connect_job.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
connect_job.add_argument('-s', '--show-progress',
action='store_true',
default=False,
help='writes the progress out to stderr on how '
'many points were streamed thus far')
connect_job.add_argument('--retry',
type=int,
default=0,
help='retry running the program N times,'
'default 0. Use -1 to retry forever.')
connect_job.add_argument('--retry-delay',
type=int,
default=10,
help='number of seconds to wait between retries.')
connect_job.add_argument('-f', '--format',
default='json',
help='available formats are json, text, csv with '
'default: json')
# programs commands
programs_parser = commands.add_parser('programs',
help='programs management')
programs_commands = programs_parser.add_subparsers(dest='programs_subcommand')
list_programs = programs_commands.add_parser('list',
help='list programs')
list_programs.add_argument('-d', '--deployment',
default=None,
help='specify the deployment name')
list_programs.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
list_programs.add_argument('-f', '--format',
default='table',
help='available formats are text, table with '
'default: table')
list_programs.add_argument('--all',
default=False,
help='list all programs, default is to list your'
' own programs')
run_programs = programs_commands.add_parser('run',
help='run a program in your local browser')
run_programs.add_argument('program_name',
help='specify the program name you wish to kick off')
pull_programs = programs_commands.add_parser('pull',
help='pull programs')
pull_programs.add_argument('directory',
help='directory to pull remote programs into')
pull_programs.add_argument('-d', '--deployment',
default=None,
help='specify the deployment name')
pull_programs.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
pull_programs.add_argument('-p', '--per-user-directory',
action='store_true',
default=False,
help='save the programs per user into a '
'separate directory')
pull_programs.add_argument('--all',
action='store_true',
default=False,
help='pull all programs, default is to list your'
' own programs')
push_programs = programs_commands.add_parser('push',
help='push programs')
push_programs.add_argument('directory',
help='directory to pick up programs to push to '
'the running Jut instance')
push_programs.add_argument('-d', '--deployment',
default=None,
help='specify the deployment name')
push_programs.add_argument('-a', '--app-url',
default=defaults.APP_URL,
help='app url (default: https://app.jut.io INTERNAL USE)')
push_programs.add_argument('--all',
default=False,
help='pull all programs, default is to list your'
' own programs')
# upload commands
upload_parser = commands.add_parser('upload',
help='upload local JSON file(s) to Jut')
if sys.stdin.isatty():
upload_parser.add_argument('source',
help='The name of a JSON file or directory '
'containing JSON files to process')
upload_parser.add_argument('-u', '--url',
help='The URL to POST data points to, if none is '
'specified we will push to the webhook for '
'the default configuration')
upload_parser.add_argument('-d', '--deployment',
dest='deployment',
default=None,
help='specify the deployment name')
upload_parser.add_argument('-s', '--space',
dest='space',
default='default',
help='specify the destination space')
upload_parser.add_argument('--dry-run',
action='store_true',
dest='dry_run',
default=False,
help='Just log the data that would have been '
'POSTed to the specified URL.')
upload_parser.add_argument('--batch-size',
dest='batch_size',
default=100,
type=int,
help='Maximum set of data points to send in each '
'POST, default: 100.')
upload_parser.add_argument('--anonymize-fields',
metavar='field_name',
dest='anonymize_fields',
nargs='+',
default=[],
help='space separated field names to anonymize '
'in the data before uploading. Currently '
'we anonymize hashing the field value with '
'md5 hash')
upload_parser.add_argument('--remove-fields',
metavar='field_name',
dest='remove_fields',
nargs='+',
default=[],
help='space separated field names to remove '
'from the data before uploading')
upload_parser.add_argument('--rename-fields',
metavar='field_name=new_field_name',
dest='rename_fields',
type=parse_key_value,
nargs='+',
default=[],
help='space separated field names to rename '
'from the data before uploading.')
# run parser
run_parser = commands.add_parser('run',
help='run juttle program from the import '
'command line')
run_parser.add_argument('juttle',
help='juttle program to execute or the filename '
'of a juttle program.')
run_parser.add_argument('-d', '--deployment',
dest='deployment',
default=None,
help='specify the deployment name')
run_parser.add_argument('-f', '--format',
default='json',
help='available formats are json, text, csv with '
'default: json')
run_parser.add_argument('-n', '--name',
help='give your program a name to appear in the '
'Jobs application')
run_parser.add_argument('-p', '--persist',
action='store_true',
default=False,
help='allow the program containing background '
'outputs to become a persistent job by '
'disconnecting form the running job (ie '
'essentially backgrounding your program)')
run_parser.add_argument('-s', '--show-progress',
action='store_true',
default=False,
help='writes the progress out to stderr on how '
'many points were streamed thus far')
run_parser.add_argument('--retry',
type=int,
default=0,
help='retry running the program N times,'
'default 0. Use -1 to retry forever.')
run_parser.add_argument('--retry-delay',
type=int,
default=10,
help='number of seconds to wait between retries.')
options = parser.parse_args()
try:
if options.subcommand == 'config':
if options.config_subcommand == 'list':
config.show()
elif options.config_subcommand == 'add':
configs.add_configuration(options)
elif options.config_subcommand == 'rm':
configs.rm_configuration(options)
elif options.config_subcommand == 'defaults':
configs.change_defaults(options)
else:
raise Exception('Unexpected config subcommand "%s"' % options.command)
elif options.subcommand == 'jobs':
if options.jobs_subcommand == 'list':
jobs.list(options)
elif options.jobs_subcommand == 'kill':
jobs.kill(options)
elif options.jobs_subcommand == 'connect':
jobs.connect(options)
else:
raise Exception('Unexpected jobs subcommand "%s"' % options.command)
elif options.subcommand == 'programs':
if options.programs_subcommand == 'list':
programs.list(options)
elif options.programs_subcommand == 'pull':
programs.pull(options)
elif options.programs_subcommand == 'push':
programs.push(options)
elif options.programs_subcommand == 'run':
programs.run(options)
else:
raise Exception('Unexpected programs subcommand "%s"' % options.command)
elif options.subcommand == 'upload':
upload.upload_file(options)
elif options.subcommand == 'run':
run.run_juttle(options)
else:
raise Exception('Unexpected jut command "%s"' % options.command)
except JutException as exception:
if is_debug_enabled():
traceback.print_exc()
error(str(exception))
sys.exit(255)