def get_webhook_url(space): """ """ configuration = config.get_default() deployment_name = configuration['deployment_name'] app_url = configuration['app_url'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) import_url = data_engine.get_import_data_url(deployment_name, token_manager=token_manager, app_url=app_url) api_key = deployments.get_apikey(deployment_name, token_manager=token_manager, app_url=app_url) return '%s/api/v1/import/webhook/?space=%s&data_source=webhook&apikey=%s' % \ (import_url, space, api_key)
def test_config_add_interactive(self): """ verify that you can add a user interactively using `jut config add` """ with temp_jut_tools_home(): configuration = config.get_default() app_url = configuration["app_url"] # -s because getpass reads from the tty and screws up automated testing process = jut("config", "add", "-a", app_url, "-s") process.expect_output("Username: "******"jut-tools-user01\n") process.expect_output("Password: "******"bigdata\n") process.expect_status(0) # check the `jut config list` shows the right output process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, "jut-tools-user01", app_url, default=True)) process.expect_eof()
def create_user_in_default_deployment(name, username, email, password): """ """ configuration = config.get_default() app_url = configuration['app_url'] deployment_name = configuration['deployment_name'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) delete_user_from_default_deployment(username, password) accounts.create_user(name, username, email, password, token_manager=token_manager, app_url=app_url) deployments.add_user(username, deployment_name, token_manager=token_manager, app_url=app_url)
def test_config_rm_non_default_non_interactive(self): """ verify you can non interactively remove a non default configuration """ with temp_jut_tools_home(): jut_user = os.environ.get("JUT_USER") jut_pass = os.environ.get("JUT_PASS") configuration = config.get_default() app_url = configuration["app_url"] process = jut("config", "add", "-u", jut_user, "-p", jut_pass, "-a", app_url) process.expect_status(0) process = jut("config", "add", "-u", "jut-tools-user01", "-p", "bigdata", "-a", app_url) process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=False)) process = jut("config", "rm", "-u", "jut-tools-user01", "-a", app_url) process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_eof()
def list(options): """ show all currently running jobs """ configuration = config.get_default() app_url = configuration['app_url'] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration['deployment_name'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) jobs = data_engine.get_jobs(deployment_name, token_manager=token_manager, app_url=app_url) if len(jobs) == 0: error('No running jobs') else: _print_jobs(jobs, token_manager, app_url, options)
def list(options): """ list programs that belong to the authenticated user """ configuration = config.get_default() app_url = configuration["app_url"] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration["deployment_name"] client_id = configuration["client_id"] client_secret = configuration["client_secret"] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) if options.all == True: account_id = None else: account_id = accounts.get_logged_in_account_id(token_manager=token_manager, app_url=app_url) programs_details = programs.get_programs( deployment_name, token_manager=token_manager, created_by=account_id, app_url=app_url ) account_ids = set() for program in programs_details: account_ids.add(program["createdBy"]) accounts_details = accounts.get_accounts(account_ids, token_manager=token_manager, app_url=app_url) account_lookup = {} for account in accounts_details["accounts"]: account_lookup[account["id"]] = account headers = ["Name", "Last Saved", "Created By"] table = [] for program in programs_details: username = account_lookup[program["createdBy"]]["username"] program_name = program["name"] last_edited = program["lastEdited"] table.append([program_name, last_edited, username]) if options.format == "table": info(tabulate.tabulate(table, headers, tablefmt="orgtbl")) elif options.format == "text": info(tabulate.tabulate(table, headers, tablefmt="orgtbl", stralign="center")) else: raise JutException('Unsupported format "%s"' % options.format)
def test_jut_jobs_connect_on_persistent_job(self): """ verify we can reconnect to a persistent job """ with temp_jut_tools_home(): configuration = config.get_default() app_url = configuration['app_url'] process = jut('config', 'add', '-u', 'jut-tools-user03', '-p', 'bigdata', '-a', app_url, '-d') process.expect_status(0) process = jut('run', '--name', 'Persistent Job #3', '-p', 'emit -limit 10000 ' '| put source_type="event", foo="bar"' '| (write -space "%s"; keep foo | pass)' % JutJobsTests.test_space) process.expect_status(0) job_id = process.read_output().strip() process = jut('jobs', 'connect', job_id, '-f', 'text') # verify that we output 'bar' a few times process.expect_output('bar\n') process.expect_output('bar\n') process.expect_output('bar\n') process.expect_output('bar\n') process.expect_output('bar\n') process.send_signal(signal.SIGTERM) process.expect_status(-signal.SIGTERM) process = jut('jobs', 'kill', job_id, '-y') process.expect_status(0) process.expect_eof() process = jut('jobs', 'list') process.expect_status(0) process.expect_error('No running jobs')
def test_config_rm_default_interactive(self): """ verify you can interactively remove a default configuration and be prompted to pick a new default """ with temp_jut_tools_home(): jut_user = os.environ.get("JUT_USER") jut_pass = os.environ.get("JUT_PASS") configuration = config.get_default() app_url = configuration["app_url"] process = jut("config", "add", "-u", jut_user, "-p", jut_pass, "-a", app_url) process.expect_status(0) process = jut("config", "add", "-u", "jut-tools-user01", "-p", "bigdata", "-a", app_url) process.expect_status(0) process = jut("config", "add", "-u", "jut-tools-user02", "-p", "bigdata", "-a", app_url) process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=False)) process.expect_output(confline(3, "jut-tools-user02", app_url, default=False)) process = jut("config", "rm") process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=False)) process.expect_output(confline(3, "jut-tools-user02", app_url, default=False)) process.expect_output("Which configuration to remove: ") process.send("1\n") process.expect_output("Pick a default configuration from the list below\n") process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, "jut-tools-user01", app_url, default=False)) process.expect_output(confline(2, "jut-tools-user02", app_url, default=False)) process.expect_output("Set default configuration to: ") process.send("2\n") process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, "jut-tools-user01", app_url, default=False)) process.expect_output(confline(2, "jut-tools-user02", app_url, default=True)) process.expect_eof()
def delete_space_from_default_deployment(space_name): configuration = config.get_default() deployment_name = configuration['deployment_name'] app_url = configuration['app_url'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) deployments.delete_space(deployment_name, space_name, token_manager=token_manager, app_url=app_url)
def test_config_list(self): """ list the saved configuration and verify it only contains the single test account with JUT_USER used by the jut unittests """ jut_user = os.environ.get("JUT_USER") configuration = config.get_default() app_url = configuration["app_url"] process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_eof()
def test_jut_kill_on_persistent_job(self): """ verify that a persistent job can be killed using `jut jobs kill` """ with temp_jut_tools_home(): configuration = config.get_default() app_url = configuration['app_url'] process = jut('config', 'add', '-u', 'jut-tools-user02', '-p', 'bigdata', '-a', app_url, '-d') process.expect_status(0) process = jut('run', '--name', 'Persistent Job #1', '-p', 'emit -limit 10000 ' '| put source_type="event" ' '| write -space "%s"' % JutJobsTests.test_space) process.expect_status(0) job_id = process.read_output().strip() process = jut('jobs', 'list', '-f', 'text') process.expect_status(0) output = process.read_output() lines = output.split('\n') re.match(r'Job ID\w+Juttle Name\w+Owner\w+Start Date\w+Persistent', lines[0]) re.match(r'%s\w+Persistent Job #1\w+jut-tools-user02.*YES' % job_id, lines[1]) process = jut('jobs', 'kill', job_id, '-y') process.expect_status(0) process.expect_eof() process = jut('jobs', 'list') process.expect_status(0) process.expect_error('No running jobs')
def test_config_add_non_interactive(self): """ add a few tests accounts to the default configuration and then add those configurations to the current jut tools config """ with temp_jut_tools_home(): configuration = config.get_default() app_url = configuration["app_url"] process = jut("config", "add", "-u", "jut-tools-user01", "-p", "bigdata", "-a", app_url) process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, "jut-tools-user01", app_url, default=True)) process.expect_eof()
def test_config_defaults_interactive(self): """ verify you can interactively modify the config default """ with temp_jut_tools_home() as home_dir: jut_user = os.environ.get("JUT_USER") jut_pass = os.environ.get("JUT_PASS") configuration = config.get_default() app_url = configuration["app_url"] process = jut("config", "add", "-u", jut_user, "-p", jut_pass, "-a", app_url) process.expect_status(0) process = jut("config", "add", "-u", "jut-tools-user01", "-p", "bigdata", "-a", app_url) process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=False)) process = jut("config", "defaults") process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=False)) process.expect_output("Set default configuration to: ") process.send("2\n") process.expect_output("Configuration updated at %s\n" % home_dir) process.expect_status(0) process.expect_eof() process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=False)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=True)) process.expect_eof()
def kill(options): """ kill a specific job by id """ configuration = config.get_default() app_url = configuration['app_url'] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration['deployment_name'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) job_details = data_engine.get_job_details(options.job_id, deployment_name, token_manager=token_manager, app_url=app_url) options.format = 'table' if options.yes: decision = 'Y' else: _print_jobs([job_details], token_manager, app_url, options) decision = prompt('Are you sure you want to delete the above job? (Y/N)') if decision == 'Y': data_engine.delete_job(options.job_id.strip(), deployment_name, token_manager=token_manager, app_url=app_url) else: raise JutException('Unexpected option "%s"' % decision)
def upload_file(options): if not sys.stdin.isatty(): json_file = sys.stdin else: json_file = open(options.source, 'r') url = options.url if url == None: configuration = config.get_default() app_url = configuration['app_url'] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration['deployment_name'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) url = integrations.get_webhook_url(deployment_name, space=options.space, token_manager=token_manager, app_url=app_url) info('Pushing to %s' % url) push_json_file(json_file, url, dry_run=options.dry_run, batch_size=options.batch_size, anonymize_fields=options.anonymize_fields, remove_fields=options.remove_fields, rename_fields=options.rename_fields)
def test_jut_jobs_with_running_jobs(self): """ verify that a simple long running program can be displayed correctly when running `jut jobs list` """ with temp_jut_tools_home(): configuration = config.get_default() app_url = configuration['app_url'] process = jut('config', 'add', '-u', 'jut-tools-user01', '-p', 'bigdata', '-a', app_url, '-d') process.expect_status(0) process = jut('run', '--name', 'Persistent Job #1', '-p', 'emit -limit 100 ' '| put source_type="event" ' '| write -space "%s"' % JutJobsTests.test_space) process.expect_status(0) job_id = process.read_output() process = jut('jobs', 'list', '-f', 'text') process.expect_status(0) output = process.read_output() lines = output.split('\n') re.match(r'Job ID\w+Juttle Name\w+Owner\w+Start Date\w+Persistent', lines[0]) re.match(r'%s\w+Persistent Job #1\w+jut-tools-user01.*YES' % job_id, lines[1])
def test_config_defaults_non_interactive(self): """ see that you can easily change the default configuration in use and that it switches back once you delete the default configuration """ with temp_jut_tools_home() as home_dir: jut_user = os.environ.get("JUT_USER") jut_pass = os.environ.get("JUT_PASS") configuration = config.get_default() app_url = configuration["app_url"] process = jut("config", "add", "-u", jut_user, "-p", jut_pass, "-a", app_url) process.expect_status(0) process = jut("config", "add", "-u", "jut-tools-user01", "-p", "bigdata", "-a", app_url) process.expect_status(0) process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=True)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=False)) process.expect_eof() process = jut("config", "defaults", "-u", "jut-tools-user01", "-a", app_url) process.expect_status(0) process.expect_output("Configuration updated at %s\n" % home_dir) process.expect_eof() process = jut("config", "list") process.expect_status(0) process.expect_output("Available jut configurations:\n") process.expect_output(confline(1, jut_user, app_url, default=False)) process.expect_output(confline(2, "jut-tools-user01", app_url, default=True)) process.expect_eof()
def delete_user_from_default_deployment(username, password): """ """ configuration = config.get_default() app_url = configuration['app_url'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) if accounts.user_exists(username, token_manager=token_manager, app_url=app_url): delete_token_manager = auth.TokenManager(username=username, password=password, app_url=app_url) accounts.delete_user(username, token_manager=delete_token_manager, app_url=app_url)
def create_space_in_default_deployment(space_name): configuration = config.get_default() deployment_name = configuration['deployment_name'] app_url = configuration['app_url'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) if deployments.space_exists(deployment_name, space_name, token_manager=token_manager, app_url=app_url): delete_space_from_default_deployment(space_name) deployments.create_space(deployment_name, space_name, token_manager=token_manager, app_url=app_url) time.sleep(SPACE_CREATE_TIMEOUT)
def pull(options): """ pull all remote programs to a local directory """ configuration = config.get_default() app_url = configuration["app_url"] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration["deployment_name"] client_id = configuration["client_id"] client_secret = configuration["client_secret"] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) if options.all == True: account_id = None else: account_id = accounts.get_logged_in_account_id(token_manager=token_manager, app_url=app_url) programs_details = programs.get_programs( deployment_name, token_manager=token_manager, created_by=account_id, app_url=app_url ) if not os.path.exists(options.directory): os.mkdir(options.directory) account_ids = set() for program in programs_details: account_ids.add(program["createdBy"]) accounts_details = accounts.get_accounts(account_ids, token_manager=token_manager, app_url=app_url) account_lookup = {} for account in accounts_details["accounts"]: account_lookup[account["id"]] = account decision = None for program in programs_details: program_name = program["name"] juttle_filename = "%s.juttle" % escape_filename(program_name) if options.per_user_directory: username = account_lookup[program["createdBy"]]["username"] userdir = os.path.join(options.directory, username) if not os.path.exists(userdir): os.mkdir(userdir) juttle_filepath = os.path.join(userdir, juttle_filename) else: juttle_filepath = os.path.join(options.directory, juttle_filename) if os.path.exists(juttle_filepath) and decision != "A": program_code = None with codecs.open(juttle_filepath, "r", encoding="UTF-8") as program_file: program_code = program_file.read() local_last_edited = int(os.stat(juttle_filepath).st_mtime) remote_last_edited = dates.iso8601_to_epoch(program["lastEdited"]) if local_last_edited != remote_last_edited: info('Juttle changed since last pull for "%s"' % program_name) decision = console.prompt( "Would you like to " "(O - Override," " S - Skip," " R - Review Changes," " A - override All)?" ) if decision == "R": info("Following is what would change if we overrode using your copy:") info("*" * 80) for line in difflib.ndiff(program["code"].split("\n"), program_code.split("\n")): info(line) info("*" * 80) decision = console.prompt("Would you like to " "(O - Override," " S - Skip)?") if decision == "S": # jump to the next file continue elif decision == "O": pass elif decision == "A": pass else: raise JutException('Unexpected option "%s"' % decision) info('importing program "%s" to %s' % (program["name"], juttle_filepath)) with codecs.open(juttle_filepath, "w", encoding="UTF-8") as program_file: program_file.write(program["code"]) # update creation time to match the lastEdited field epoch = dates.iso8601_to_epoch(program["lastEdited"]) os.utime(juttle_filepath, (epoch, epoch))
def connect(options): options.persist = False if not config.is_configured(): configs.add_configuration(options) configuration = config.get_default() app_url = configuration['app_url'] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration['deployment_name'] client_id = configuration['client_id'] client_secret = configuration['client_secret'] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) total_points = 0 def show_progress(): if options.show_progress: error('streamed %s points', total_points, end='\r') def show_error_or_warning(data): """ handle error and warning reporting """ if 'error' in data: prefix = 'Error' elif 'warning' in data: prefix = 'Warning' else: raise Exception('Unexpected error/warning received %s' % data) message = None location = None if 'context' in data: message = data['context']['message'] # not all errors or warnings have location information if 'location' in data['context']['info']: location = data['context']['info']['location'] line = location['start']['line'] column = location['start']['column'] else: message = '%s: %s' % (prefix, message) if location != None: error('%s line %s, column %s of %s: %s' % (prefix, line, column, location['filename'], message)) else: error(message) if options.format == 'json': formatter = JSONFormatter(options) elif options.format == 'text': formatter = TextFormatter(options) elif options.format == 'csv': formatter = CSVFormatter(options) else: raise JutException('Unsupported output format "%s"' % options.format) job_id = options.job_id done = False with_errors = False max_retries = options.retry retry_delay = options.retry_delay retry = 0 while not done: try: if not options.persist: formatter.start() for data in data_engine.connect_job(job_id, deployment_name, token_manager=token_manager, app_url=app_url): show_progress() if 'job' in data: # job details if options.persist: # lets print the job id info(data['job']['id']) if 'points' in data: points = data['points'] for point in points: formatter.point(point) total_points += len(points) elif 'error' in data: show_error_or_warning(data) with_errors = True elif 'warning' in data: show_error_or_warning(data) done = True except JutException: retry += 1 if max_retries != -1 and retry > max_retries: raise time.sleep(retry_delay) finally: if options.show_progress: # one enter to retain the last value of progress output info('') if not options.persist: formatter.stop() if with_errors: raise JutException('Error while running juttle')
def push(options): configuration = config.get_default() app_url = configuration["app_url"] if options.deployment != None: deployment_name = options.deployment else: deployment_name = configuration["deployment_name"] client_id = configuration["client_id"] client_secret = configuration["client_secret"] token_manager = auth.TokenManager(client_id=client_id, client_secret=client_secret, app_url=app_url) if not os.path.exists(options.source): raise JutException('Source "%s" does not exists.') filenames = [] if os.path.isdir(options.source): for filename in os.listdir(options.source): if filename.endswith(".juttle"): filenames.append(filename) else: filenames.append(options.source) decision = None for filename in filenames: filepath = os.path.join(options.source, filename) program_name = urllib.unquote_plus(os.path.basename(filepath).replace(r".juttle", "")) info('Found program "%s"' % program_name) with codecs.open(filepath, "r", encoding="UTF-8") as program_file: program_code = program_file.read() local_last_edited = int(os.stat(filepath).st_mtime) if programs.program_exists(program_name, deployment_name, token_manager=token_manager, app_url=app_url): # one last safety to check if the modification time of # the file still matches the lastEdited of the existing # copy on Jut otherwise we prompt the user for confirmation program = programs.get_program(program_name, deployment_name, token_manager=token_manager, app_url=app_url) remote_last_edited = dates.iso8601_to_epoch(program["lastEdited"]) if local_last_edited != remote_last_edited and decision != "A": info('Juttle changed since last pull for "%s"' % program_name) decision = console.prompt( "Would you like to " "(O - Override," " S - Skip," " R - Review Changes," " A - override All)?" ) if decision == "R": info("Following is what would change if we overrode using your copy:") info("*" * 80) for line in difflib.ndiff(program["code"].split("\n"), program_code.split("\n")): info(line) info("*" * 80) decision = console.prompt("Would you like to " "(O - Override," " S - Skip)?") if decision == "S": # jump to the next file continue elif decision == "O": pass elif decision == "A": pass else: raise JutException('Unexpected option "%s"' % decision) last_edited_iso = dates.epoch_to_iso8601(local_last_edited) programs.update_program( program_name, program_code, deployment_name, last_edited=last_edited_iso, token_manager=token_manager, app_url=app_url, ) os.utime(filepath, (local_last_edited, local_last_edited)) else: last_edited_iso = dates.epoch_to_iso8601(local_last_edited) programs.save_program( program_name, program_code, deployment_name, last_edited=last_edited_iso, token_manager=token_manager, app_url=app_url, ) os.utime(filepath, (local_last_edited, local_last_edited))