def __wss_connect(data_url, token_manager, job_id=None): """ Establish the websocket connection to the data engine. When job_id is provided we're basically establishing a websocket to an existing program that was already started using the jobs API job_id: job id of a running program """ url = '%s/api/v1/juttle/channel' % data_url.replace('https://', 'wss://') token_obj = { "accessToken": token_manager.get_access_token() } if job_id != None: token_obj['job_id'] = job_id if is_debug_enabled(): debug("connecting to %s", url) websocket = create_connection(url) websocket.settimeout(10) if is_debug_enabled(): debug("sent %s", json.dumps(token_obj)) websocket.send(json.dumps(token_obj)) return websocket
def get_access_token(self): """ get a valid access token """ if self.is_access_token_expired(): if is_debug_enabled(): debug('requesting new access_token') token = get_access_token(username=self.username, password=self.password, client_id=self.client_id, client_secret=self.client_secret, app_url=self.app_url) # lets make sure to refresh before we're halfway to expiring self.expires_at = time.time() + token['expires_in']/2 self.access_token = token['access_token'] return self.access_token
def run(juttle, deployment_name, program_name=None, persist=False, token_manager=None, app_url=defaults.APP_URL): """ run a juttle program through the juttle streaming API and return the various events that are part of running a Juttle program which include: * Initial job status details including information to associate multiple flowgraphs with their individual outputs (sinks): { "status": "ok", "job": { "channel_id": "56bde5f0", "_start_time": "2015-10-03T06:59:49.233Z", "alias": "jut-tools program 1443855588", "_ms_begin": 1443855589233, "user": "******", "timeout": 5, "id": "b973bce6" }, "now": "2015-10-03T06:59:49.230Z", "stats": ... "sinks": [ { "location": { "start": { "column": 17, "line": 1, "offset": 16 }, "end": { "column": 24, "line": 1, "offset": 23 }, "filename": "main" }, "name": "table", "channel": "sink237", "options": { "_jut_time_bounds": [] } }, ... as many sinks as there are flowgrpahs in your program ] } * Each set of points returned along with the indication of which sink they belong to: { "points": [ array of points ], "sink": sink_id } * Error event indicating where in your program the error occurred { "error": true, payload with "info" and "context" explaining exact error } * Warning event indicating where in your program the error occurred { "warning": true, payload with "info" and "context" explaining exact warning } * ... juttle: juttle program to execute deployment_name: the deployment name to execute the program on persist: if set to True then we won't wait for response data and will disconnect from the websocket leaving the program running in the background if it is uses a background output (http://docs.jut.io/juttle-guide/#background_outputs) and therefore becomes a persistent job. token_manager: auth.TokenManager object app_url: optional argument used primarily for internal Jut testing """ headers = token_manager.get_access_token_headers() data_url = get_juttle_data_url(deployment_name, app_url=app_url, token_manager=token_manager) websocket = __wss_connect(data_url, token_manager) data = websocket.recv() channel_id_obj = json.loads(data) if is_debug_enabled(): debug('got channel response %s', json.dumps(channel_id_obj)) channel_id = channel_id_obj['channel_id'] juttle_job = { 'channel_id': channel_id, 'alias': program_name, 'program': juttle } response = requests.post('%s/api/v1/jobs' % data_url, data=json.dumps(juttle_job), headers=headers) if response.status_code != 200: yield { "error": True, "context": response.json() } return job_info = response.json() # yield job_info so the caller to this method can figure out which sinks # correlate to which flowgraphs yield job_info job_id = job_info['job']['id'] if is_debug_enabled(): debug('started job %s', json.dumps(job_info)) for data in connect_job(job_id, deployment_name, token_manager=token_manager, app_url=app_url, persist=persist, websocket=websocket, data_url=data_url): yield data
def connect_job(job_id, deployment_name, token_manager=None, app_url=defaults.APP_URL, persist=False, websocket=None, data_url=None): """ connect to a running Juttle program by job_id """ if data_url == None: data_url = get_data_url_for_job(job_id, deployment_name, token_manager=token_manager, app_url=app_url) if websocket == None: websocket = __wss_connect(data_url, token_manager, job_id=job_id) pong = json.dumps({ 'pong': True }) if not persist: job_finished = False while not job_finished: try: data = websocket.recv() if data: payload = json.loads(data) if is_debug_enabled(): printable_payload = dict(payload) if 'points' in payload: # don't want to print out all the outputs when in # debug mode del printable_payload['points'] printable_payload['points'] = 'NOT SHOWN' debug('received %s' % json.dumps(printable_payload)) if 'ping' in payload.keys(): # ping/pong (ie heartbeat) mechanism websocket.send(pong) if is_debug_enabled(): debug('sent %s' % json.dumps(pong)) if 'job_end' in payload.keys() and payload['job_end'] == True: job_finished = True if token_manager.is_access_token_expired(): debug('refreshing access token') token_obj = { "accessToken": token_manager.get_access_token() } # refresh authentication token websocket.send(json.dumps(token_obj)) if 'error' in payload: if payload['error'] == 'NONEXISTENT-JOB': raise JutException('Job "%s" no longer running' % job_id) # return all channel messages yield payload else: debug('payload was "%s", forcing websocket reconnect' % data) raise IOError() except IOError: if is_debug_enabled(): traceback.print_exc() # # We'll retry for just under 30s since internally we stop # running non persistent programs after 30s of not heartbeating # with the client # retry = 1 while retry <= 5: try: debug('network error reconnecting to job %s, ' 'try %s of 5' % (job_id, retry)) websocket = __wss_connect(data_url, token_manager, job_id=job_id) break except socket.error: if is_debug_enabled(): traceback.print_exc() retry += 1 time.sleep(5) debug('network error reconnecting to job %s, ' 'try %s of 5' % (job_id, retry)) websocket = __wss_connect(data_url, token_manager, job_id=job_id) websocket.close()