def __wss_connect(data_url, token_manager, job_id=None): """ Establish the websocket connection to the data engine. When job_id is provided we're basically establishing a websocket to an existing program that was already started using the jobs API job_id: job id of a running program """ url = '%s/api/v1/juttle/channel' % data_url.replace('https://', 'wss://') token_obj = { "accessToken": token_manager.get_access_token() } if job_id != None: token_obj['job_id'] = job_id if is_debug_enabled(): debug("connecting to %s", url) websocket = create_connection(url) websocket.settimeout(10) if is_debug_enabled(): debug("sent %s", json.dumps(token_obj)) websocket.send(json.dumps(token_obj)) return websocket
def get_access_token(self): """ get a valid access token """ if self.is_access_token_expired(): if is_debug_enabled(): debug('requesting new access_token') token = get_access_token(username=self.username, password=self.password, client_id=self.client_id, client_secret=self.client_secret, app_url=self.app_url) # lets make sure to refresh before we're halfway to expiring self.expires_at = time.time() + token['expires_in']/2 self.access_token = token['access_token'] return self.access_token
def run(juttle, deployment_name, program_name=None, persist=False, token_manager=None, app_url=defaults.APP_URL): """ run a juttle program through the juttle streaming API and return the various events that are part of running a Juttle program which include: * Initial job status details including information to associate multiple flowgraphs with their individual outputs (sinks): { "status": "ok", "job": { "channel_id": "56bde5f0", "_start_time": "2015-10-03T06:59:49.233Z", "alias": "jut-tools program 1443855588", "_ms_begin": 1443855589233, "user": "******", "timeout": 5, "id": "b973bce6" }, "now": "2015-10-03T06:59:49.230Z", "stats": ... "sinks": [ { "location": { "start": { "column": 17, "line": 1, "offset": 16 }, "end": { "column": 24, "line": 1, "offset": 23 }, "filename": "main" }, "name": "table", "channel": "sink237", "options": { "_jut_time_bounds": [] } }, ... as many sinks as there are flowgrpahs in your program ] } * Each set of points returned along with the indication of which sink they belong to: { "points": [ array of points ], "sink": sink_id } * Error event indicating where in your program the error occurred { "error": true, payload with "info" and "context" explaining exact error } * Warning event indicating where in your program the error occurred { "warning": true, payload with "info" and "context" explaining exact warning } * ... juttle: juttle program to execute deployment_name: the deployment name to execute the program on persist: if set to True then we won't wait for response data and will disconnect from the websocket leaving the program running in the background if it is uses a background output (http://docs.jut.io/juttle-guide/#background_outputs) and therefore becomes a persistent job. token_manager: auth.TokenManager object app_url: optional argument used primarily for internal Jut testing """ headers = token_manager.get_access_token_headers() data_url = get_juttle_data_url(deployment_name, app_url=app_url, token_manager=token_manager) websocket = __wss_connect(data_url, token_manager) data = websocket.recv() channel_id_obj = json.loads(data) if is_debug_enabled(): debug('got channel response %s', json.dumps(channel_id_obj)) channel_id = channel_id_obj['channel_id'] juttle_job = { 'channel_id': channel_id, 'alias': program_name, 'program': juttle } response = requests.post('%s/api/v1/jobs' % data_url, data=json.dumps(juttle_job), headers=headers) if response.status_code != 200: yield { "error": True, "context": response.json() } return job_info = response.json() # yield job_info so the caller to this method can figure out which sinks # correlate to which flowgraphs yield job_info job_id = job_info['job']['id'] if is_debug_enabled(): debug('started job %s', json.dumps(job_info)) for data in connect_job(job_id, deployment_name, token_manager=token_manager, app_url=app_url, persist=persist, websocket=websocket, data_url=data_url): yield data
def connect_job(job_id, deployment_name, token_manager=None, app_url=defaults.APP_URL, persist=False, websocket=None, data_url=None): """ connect to a running Juttle program by job_id """ if data_url == None: data_url = get_data_url_for_job(job_id, deployment_name, token_manager=token_manager, app_url=app_url) if websocket == None: websocket = __wss_connect(data_url, token_manager, job_id=job_id) pong = json.dumps({ 'pong': True }) if not persist: job_finished = False while not job_finished: try: data = websocket.recv() if data: payload = json.loads(data) if is_debug_enabled(): printable_payload = dict(payload) if 'points' in payload: # don't want to print out all the outputs when in # debug mode del printable_payload['points'] printable_payload['points'] = 'NOT SHOWN' debug('received %s' % json.dumps(printable_payload)) if 'ping' in payload.keys(): # ping/pong (ie heartbeat) mechanism websocket.send(pong) if is_debug_enabled(): debug('sent %s' % json.dumps(pong)) if 'job_end' in payload.keys() and payload['job_end'] == True: job_finished = True if token_manager.is_access_token_expired(): debug('refreshing access token') token_obj = { "accessToken": token_manager.get_access_token() } # refresh authentication token websocket.send(json.dumps(token_obj)) if 'error' in payload: if payload['error'] == 'NONEXISTENT-JOB': raise JutException('Job "%s" no longer running' % job_id) # return all channel messages yield payload else: debug('payload was "%s", forcing websocket reconnect' % data) raise IOError() except IOError: if is_debug_enabled(): traceback.print_exc() # # We'll retry for just under 30s since internally we stop # running non persistent programs after 30s of not heartbeating # with the client # retry = 1 while retry <= 5: try: debug('network error reconnecting to job %s, ' 'try %s of 5' % (job_id, retry)) websocket = __wss_connect(data_url, token_manager, job_id=job_id) break except socket.error: if is_debug_enabled(): traceback.print_exc() retry += 1 time.sleep(5) debug('network error reconnecting to job %s, ' 'try %s of 5' % (job_id, retry)) websocket = __wss_connect(data_url, token_manager, job_id=job_id) websocket.close()
def main(): class JutArgParser(argparse.ArgumentParser): """ custom argument parser so we show the full comand line help menu """ def error(self, message): error(message) self.print_help() sys.exit(2) parser = JutArgParser(description='jut - jut command line tools') commands = parser.add_subparsers(dest='subcommand') # config commands config_parser = commands.add_parser('config', help='configuration management') config_commands = config_parser.add_subparsers(dest='config_subcommand') _ = config_commands.add_parser('list', help='list configurations') defaults_config = config_commands.add_parser('defaults', help='change the configuration defaults') defaults_config.add_argument('-u', '--username', help='username to use') defaults_config.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io ' 'INTERNAL USE)') add_config = config_commands.add_parser('add', help='add another configuration ' '(default when no sub command ' 'is provided)') add_config.add_argument('-u', '--username', help='username to use') add_config.add_argument('-p', '--password', help='password to use') add_config.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') add_config.add_argument('-d', '--default', action='store_true', help='sets this configuration to the default') add_config.add_argument('-s', '--show-password', action='store_true', default=False, help='shows password as you type it interactively') rm_config = config_commands.add_parser('rm', help='remove a configuration') rm_config.add_argument('-u', '--username', help='username to use') rm_config.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') # jobs commands jobs_parser = commands.add_parser('jobs', help='jobs management') jobs_commands = jobs_parser.add_subparsers(dest='jobs_subcommand') list_jobs = jobs_commands.add_parser('list', help='list running jobs') list_jobs.add_argument('-d', '--deployment', default=None, help='specify the deployment name') list_jobs.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') list_jobs.add_argument('-f', '--format', default='table', help='available formats are text, table with ' 'default: table') kill_job = jobs_commands.add_parser('kill', help='kill running job') kill_job.add_argument('job_id', help='specify the job_id to kill') kill_job.add_argument('-d', '--deployment', default=None, help='specify the deployment name') kill_job.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') kill_job.add_argument('-y', '--yes', action='store_true', default=False, help='kill without prompting for confirmation') connect_job = jobs_commands.add_parser('connect', help='connect to a persistent job') connect_job.add_argument('job_id', help='specify the job_id to connect to') connect_job.add_argument('-d', '--deployment', default=None, help='specify the deployment name') connect_job.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') connect_job.add_argument('-s', '--show-progress', action='store_true', default=False, help='writes the progress out to stderr on how ' 'many points were streamed thus far') connect_job.add_argument('--retry', type=int, default=0, help='retry running the program N times,' 'default 0. Use -1 to retry forever.') connect_job.add_argument('--retry-delay', type=int, default=10, help='number of seconds to wait between retries.') connect_job.add_argument('-f', '--format', default='json', help='available formats are json, text, csv with ' 'default: json') # programs commands programs_parser = commands.add_parser('programs', help='programs management') programs_commands = programs_parser.add_subparsers(dest='programs_subcommand') list_programs = programs_commands.add_parser('list', help='list programs') list_programs.add_argument('-d', '--deployment', default=None, help='specify the deployment name') list_programs.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') list_programs.add_argument('-f', '--format', default='table', help='available formats are text, table with ' 'default: table') list_programs.add_argument('--all', default=False, help='list all programs, default is to list your' ' own programs') run_programs = programs_commands.add_parser('run', help='run a program in your local browser') run_programs.add_argument('program_name', help='specify the program name you wish to kick off') pull_programs = programs_commands.add_parser('pull', help='pull programs') pull_programs.add_argument('directory', help='directory to pull remote programs into') pull_programs.add_argument('-d', '--deployment', default=None, help='specify the deployment name') pull_programs.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') pull_programs.add_argument('-p', '--per-user-directory', action='store_true', default=False, help='save the programs per user into a ' 'separate directory') pull_programs.add_argument('--all', action='store_true', default=False, help='pull all programs, default is to list your' ' own programs') push_programs = programs_commands.add_parser('push', help='push programs') push_programs.add_argument('directory', help='directory to pick up programs to push to ' 'the running Jut instance') push_programs.add_argument('-d', '--deployment', default=None, help='specify the deployment name') push_programs.add_argument('-a', '--app-url', default=defaults.APP_URL, help='app url (default: https://app.jut.io INTERNAL USE)') push_programs.add_argument('--all', default=False, help='pull all programs, default is to list your' ' own programs') # upload commands upload_parser = commands.add_parser('upload', help='upload local JSON file(s) to Jut') if sys.stdin.isatty(): upload_parser.add_argument('source', help='The name of a JSON file or directory ' 'containing JSON files to process') upload_parser.add_argument('-u', '--url', help='The URL to POST data points to, if none is ' 'specified we will push to the webhook for ' 'the default configuration') upload_parser.add_argument('-d', '--deployment', dest='deployment', default=None, help='specify the deployment name') upload_parser.add_argument('-s', '--space', dest='space', default='default', help='specify the destination space') upload_parser.add_argument('--dry-run', action='store_true', dest='dry_run', default=False, help='Just log the data that would have been ' 'POSTed to the specified URL.') upload_parser.add_argument('--batch-size', dest='batch_size', default=100, type=int, help='Maximum set of data points to send in each ' 'POST, default: 100.') upload_parser.add_argument('--anonymize-fields', metavar='field_name', dest='anonymize_fields', nargs='+', default=[], help='space separated field names to anonymize ' 'in the data before uploading. Currently ' 'we anonymize hashing the field value with ' 'md5 hash') upload_parser.add_argument('--remove-fields', metavar='field_name', dest='remove_fields', nargs='+', default=[], help='space separated field names to remove ' 'from the data before uploading') upload_parser.add_argument('--rename-fields', metavar='field_name=new_field_name', dest='rename_fields', type=parse_key_value, nargs='+', default=[], help='space separated field names to rename ' 'from the data before uploading.') # run parser run_parser = commands.add_parser('run', help='run juttle program from the import ' 'command line') run_parser.add_argument('juttle', help='juttle program to execute or the filename ' 'of a juttle program.') run_parser.add_argument('-d', '--deployment', dest='deployment', default=None, help='specify the deployment name') run_parser.add_argument('-f', '--format', default='json', help='available formats are json, text, csv with ' 'default: json') run_parser.add_argument('-n', '--name', help='give your program a name to appear in the ' 'Jobs application') run_parser.add_argument('-p', '--persist', action='store_true', default=False, help='allow the program containing background ' 'outputs to become a persistent job by ' 'disconnecting form the running job (ie ' 'essentially backgrounding your program)') run_parser.add_argument('-s', '--show-progress', action='store_true', default=False, help='writes the progress out to stderr on how ' 'many points were streamed thus far') run_parser.add_argument('--retry', type=int, default=0, help='retry running the program N times,' 'default 0. Use -1 to retry forever.') run_parser.add_argument('--retry-delay', type=int, default=10, help='number of seconds to wait between retries.') options = parser.parse_args() try: if options.subcommand == 'config': if options.config_subcommand == 'list': config.show() elif options.config_subcommand == 'add': configs.add_configuration(options) elif options.config_subcommand == 'rm': configs.rm_configuration(options) elif options.config_subcommand == 'defaults': configs.change_defaults(options) else: raise Exception('Unexpected config subcommand "%s"' % options.command) elif options.subcommand == 'jobs': if options.jobs_subcommand == 'list': jobs.list(options) elif options.jobs_subcommand == 'kill': jobs.kill(options) elif options.jobs_subcommand == 'connect': jobs.connect(options) else: raise Exception('Unexpected jobs subcommand "%s"' % options.command) elif options.subcommand == 'programs': if options.programs_subcommand == 'list': programs.list(options) elif options.programs_subcommand == 'pull': programs.pull(options) elif options.programs_subcommand == 'push': programs.push(options) elif options.programs_subcommand == 'run': programs.run(options) else: raise Exception('Unexpected programs subcommand "%s"' % options.command) elif options.subcommand == 'upload': upload.upload_file(options) elif options.subcommand == 'run': run.run_juttle(options) else: raise Exception('Unexpected jut command "%s"' % options.command) except JutException as exception: if is_debug_enabled(): traceback.print_exc() error(str(exception)) sys.exit(255)