def create_cluster(request): response = {'status': -1} cluster_name = request.POST.get('cluster_name') cdh_version = request.POST.get('cdh_version') public_key = request.POST.get('public_key') instance_type = request.POST.get('instance_type', "workers_group_size" '') environment_name = request.POST.get('environment_name') workers_group_size = request.POST.get('workers_group_size', '3') namespace_name = request.POST.get('namespace_name', 'null') api = AnalyticDbApi(request.user) data = api.create_cluster(cloud_provider='aws', cluster_name=cluster_name, cdh_version=cdh_version, public_key=public_key, instance_type=instance_type, environment_name=environment_name, namespace_name=namespace_name, workers_group_size=workers_group_size) if data: response['status'] = 0 response['data'] = data else: response['message'] = 'Workload Analytics: %s' % data['details'] return JsonResponse(response)
def action(self, appid, action): message = {'message': '', 'status': 0} if action.get('action') == 'kill': api = AnalyticDbApi(self.user) for _id in appid: result = api.delete_cluster(_id) if result.get('error'): message['message'] = result.get('error') message['status'] = -1 elif result.get('contents') and message.get('status') != -1: message['message'] = result.get('contents') return message;
def get_context_namespaces(request, interface): response = {'dynamicClusters': False} namespaces = [] clusters = get_clusters(request.user).values() if interface == 'hive' or interface == 'impala' or interface == 'report': namespaces.extend([ { 'id': cluster['id'], 'name': cluster['name'], 'status': 'CREATED', 'computes': [cluster] } for cluster in clusters if cluster.get('type') == 'direct' # and interface == 'hive' ]) # From Altus SDX if [cluster for cluster in clusters if cluster['type'] == 'altus']: # Note: attaching computes to namespaces might be done via the frontend in the future if interface == 'impala': adb_clusters = AnalyticDbApi( request.user).list_clusters()['clusters'] for _cluster in adb_clusters: # Add "fake" namespace if needed if not _cluster.get('namespaceCrn'): _cluster['namespaceCrn'] = _cluster['crn'] _cluster['id'] = _cluster['crn'] _cluster['namespaceName'] = _cluster['clusterName'] _cluster['name'] = _cluster['clusterName'] else: adb_clusters = [] namespaces.extend([ { 'id': namespace.get('crn', 'None'), 'name': namespace.get('namespaceName'), 'status': namespace.get('status'), 'computes': [ _cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn') ] } for namespace in SdxApi(request.user).list_namespaces() + # Adding "fake" namespace for cluster without one [ _cluster for _cluster in adb_clusters if not cluster.get('namespaceCrn') and _cluster.get('status') == 'CREATED' ] ]) response['dynamicClusters'] = True response[interface] = namespaces response['status'] = 0 return JsonResponse(response)
def get_context_namespaces(request, interface): ''' Namespaces are node cluster contexts (e.g. Hive + Ranger) that can be queried by computes. ''' response = {} namespaces = [] clusters = list(get_clusters(request.user).values()) # Currently broken if not sent namespaces.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'status': 'CREATED', 'computes': [cluster] } for cluster in clusters if cluster.get('type') == 'direct' ]) if interface == 'hive' or interface == 'impala' or interface == 'report': if get_cluster_config(request.user)['has_computes']: # Note: attaching computes to namespaces might be done via the frontend in the future if interface == 'impala': if IS_K8S_ONLY.get(): adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] for _cluster in adb_clusters: # Add "fake" namespace if needed if not _cluster.get('namespaceCrn'): _cluster['namespaceCrn'] = _cluster['crn'] _cluster['id'] = _cluster['crn'] _cluster['namespaceName'] = _cluster['clusterName'] _cluster['name'] = _cluster['clusterName'] _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8S_ONLY.get() else '', else: adb_clusters = [] if IS_K8S_ONLY.get(): sdx_namespaces = [] else: sdx_namespaces = SdxApi(request.user).list_namespaces() # Adding "fake" namespace for cluster without one sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])]) namespaces.extend([{ 'id': namespace.get('crn', 'None'), 'name': namespace.get('namespaceName'), 'status': namespace.get('status'), 'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')] } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get() ]) response[interface] = namespaces response['status'] = 0 return JsonResponse(response)
def apps(self, filters): api = AnalyticDbApi(self.user) jobs = api.list_clusters() return { 'apps': [{ 'id': app['crn'], 'name': '%(clusterName)s' % app, 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': 'Altus %(workersGroupSize)s %(instanceType)s %(cdhVersion)s' % app, 'user': app['clusterName'].split('-', 1)[0], 'progress': 100, 'queue': 'group', 'duration': 1, 'submitted': app['creationDate'], 'canWrite': True } for app in sorted(jobs['clusters'], key=lambda a: a['creationDate'], reverse=True)], 'total': len(jobs) }
def get_context_namespaces(request, interface): response = {} namespaces = [] clusters = get_clusters(request.user).values() namespaces.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'status': 'CREATED', 'computes': [cluster] } for cluster in clusters if cluster.get('type') == 'direct' and cluster['interface'] in (interface, 'all') ]) if interface == 'hive' or interface == 'impala' or interface == 'report': # From Altus SDX if [cluster for cluster in clusters if 'altus' in cluster['type']]: # Note: attaching computes to namespaces might be done via the frontend in the future if interface == 'impala': if IS_K8S_ONLY.get(): adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] for _cluster in adb_clusters: # Add "fake" namespace if needed if not _cluster.get('namespaceCrn'): _cluster['namespaceCrn'] = _cluster['crn'] _cluster['id'] = _cluster['crn'] _cluster['namespaceName'] = _cluster['clusterName'] _cluster['name'] = _cluster['clusterName'] _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8S_ONLY.get() else '', else: adb_clusters = [] if IS_K8S_ONLY.get(): sdx_namespaces = [] else: sdx_namespaces = SdxApi(request.user).list_namespaces() # Adding "fake" namespace for cluster without one sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])]) namespaces.extend([{ 'id': namespace.get('crn', 'None'), 'name': namespace.get('namespaceName'), 'status': namespace.get('status'), 'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')] } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get() ]) response[interface] = namespaces response['status'] = 0 return JsonResponse(response)
def get_context_computes(request, interface): response = {} computes = [] clusters = get_clusters(request.user).values() if interface == 'hive' or interface == 'impala' or interface == 'oozie' or interface == 'jobs' or interface == 'report': computes.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'namespace': cluster['id'], 'interface': interface, 'type': cluster['type'] } for cluster in clusters if cluster.get('type') == 'direct']) if interface == 'impala' or interface == 'jobs' or interface == 'report': if [cluster for cluster in clusters if cluster['type'] == 'altus']: computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'namespace': cluster.get('namespaceCrn', cluster.get('crn')), 'type': 'altus-adb' } for cluster in AnalyticDbApi(request.user).list_clusters() ['clusters'] if cluster.get('status') == 'CREATED']) if interface == 'oozie' or interface == 'jobs' or interface == 'spark2': if [cluster for cluster in clusters if cluster['type'] == 'altus']: computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'environmentType': cluster.get('environmentType'), 'serviceType': cluster.get('serviceType'), 'namespace': cluster.get('namespaceCrn'), 'type': 'altus-de' } for cluster in DataEngApi(request.user).list_clusters() ['clusters']]) # TODO if interface == 'spark2' keep only SPARK type response[interface] = computes response['status'] = 0 return JsonResponse(response)
def get_context_computes(request, interface): response = {} computes = [] clusters = get_clusters(request.user).values() has_altus_clusters = [cluster for cluster in clusters if 'altus' in cluster['type']] computes.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'namespace': cluster['id'], 'interface': interface, 'type': cluster['type'] } for cluster in clusters if cluster.get('type') == 'direct' and cluster['interface'] in (interface, 'all') ]) if has_altus_clusters: if interface == 'impala' or interface == 'report': if IS_K8S_ONLY.get(): dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: dw_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'namespace': cluster.get('namespaceCrn', cluster.get('crn')), 'compute_end_point': IS_K8S_ONLY.get() and '%(publicHost)s' % cluster['coordinatorEndpoint'] or '', 'type': 'altus-dw' } for cluster in dw_clusters if (cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515') or (IS_K8S_ONLY.get() and 'TERMINAT' not in cluster['status'])] ) if interface == 'oozie' or interface == 'spark2': computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'environmentType': cluster.get('environmentType'), 'serviceType': cluster.get('serviceType'), 'namespace': cluster.get('namespaceCrn'), 'type': 'altus-de' } for cluster in DataEngApi(request.user).list_clusters()['clusters']] ) # TODO if interface == 'spark2' keep only SPARK type response[interface] = computes response['status'] = 0 return JsonResponse(response)
def __init__(self, user, version=1): super(DataWarehouseClusterApi, self).__init__(user) self.version = version self.api = DataWarehouse2Api( self.user) if version == 2 else AnalyticDbApi(self.user)
def __init__(self, user, cluster_crn): self.user = user self.cluster_crn = cluster_crn self.api = AnalyticDbApi(self.user)
class HueQuery(): def __init__(self, user, cluster_crn): self.user = user self.cluster_crn = cluster_crn self.api = AnalyticDbApi(self.user) def do_post(self, url_path): payload = '''{"method":"POST","url":"https://localhost:8888''' + url_path + '''","httpVersion":"HTTP/1.1","headers":[{"name":"Accept-Encoding","value":"gzip, deflate, br"},{"name":"Content-Type","value":"application/x-www-form-urlencoded; charset=UTF-8"},{"name":"Accept","value":"*/*"},{"name":"X-Requested-With","value":"XMLHttpRequest"},{"name":"Connection","value":"keep-alive"}],"queryString":[],"postData": { "mimeType": "application/x-www-form-urlencoded; charset=UTF-8", "text": "snippet=%7B%22type%22%3A%22impala%22%2C%22source%22%3A%22data%22%7D", "params": [ { "name": "snippet", "value": "%7B%22type%22%3A%22impala%22%2C%22source%22%3A%22data%22%7D" } ] }}''' resp = self.api.submit_hue_query(self.cluster_crn, payload) return json.loads(resp['payload']) def do_execute(self, query): payload = ''' { "method": "POST", "url": "http://127.0.0.1:8000/notebook/api/execute/impala", "httpVersion": "HTTP/1.1", "headers": [ { "name": "Accept-Encoding", "value": "gzip, deflate, br" }, { "name": "Content-Type", "value": "application/x-www-form-urlencoded; charset=UTF-8" }, { "name": "Accept", "value": "*/*" }, { "name": "X-Requested-With", "value": "XMLHttpRequest" }, { "name": "Connection", "value": "keep-alive" } ], "queryString": [], "cookies": [ ], "postData": { "mimeType": "application/x-www-form-urlencoded; charset=UTF-8", "text": "notebook=%7B%22uuid%22%3A%22f2b8a233-c34c-44b8-a8a1-0e6123996216%22%2C%22name%22%3A%22%22%2C%22description%22%3A%22%22%2C%22type%22%3A%22query-impala%22%2C%22initialType%22%3A%22impala%22%2C%22coordinatorUuid%22%3Anull%2C%22isHistory%22%3Atrue%2C%22isManaged%22%3Afalse%2C%22parentSavedQueryUuid%22%3Anull%2C%22isSaved%22%3Afalse%2C%22onSuccessUrl%22%3Anull%2C%22pubSubUrl%22%3Anull%2C%22isPresentationModeDefault%22%3Afalse%2C%22isPresentationMode%22%3Afalse%2C%22isPresentationModeInitialized%22%3Atrue%2C%22presentationSnippets%22%3A%7B%7D%2C%22isHidingCode%22%3Afalse%2C%22snippets%22%3A%5B%7B%22id%22%3A%22dd5755a3-e8db-82d9-4f98-9f4fb5a99a06%22%2C%22name%22%3A%22%22%2C%22type%22%3A%22impala%22%2C%22isBatchable%22%3Atrue%2C%22aceCursorPosition%22%3A%7B%22column%22%3A33%2C%22row%22%3A0%7D%2C%22errors%22%3A%5B%5D%2C%22aceErrorsHolder%22%3A%5B%5D%2C%22aceWarningsHolder%22%3A%5B%5D%2C%22aceErrors%22%3A%5B%5D%2C%22aceWarnings%22%3A%5B%5D%2C%22editorMode%22%3Atrue%2C%22dbSelectionVisible%22%3Afalse%2C%22isSqlDialect%22%3Atrue%2C%22namespaceRefreshEnabled%22%3Afalse%2C%22availableNamespaces%22%3A%5B%5D%2C%22availableComputes%22%3A%5B%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22default-romain%22%2C%22id%22%3A%22default-romain%22%2C%22name%22%3A%22default-romain%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22compute1%22%2C%22id%22%3A%22compute1%22%2C%22name%22%3A%22compute1%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22compute2%22%2C%22id%22%3A%22compute2%22%2C%22name%22%3A%22compute2%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22altus%22%2C%22namespace%22%3A%22Altus%22%2C%22id%22%3A%22Altus%22%2C%22name%22%3A%22Altus%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22storage1%22%2C%22id%22%3A%22storage1%22%2C%22name%22%3A%22storage1%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22storage2%22%2C%22id%22%3A%22storage2%22%2C%22name%22%3A%22storage2%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22Default%22%2C%22id%22%3A%22Default%22%2C%22name%22%3A%22default%22%7D%5D%2C%22compute%22%3A%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22default-romain%22%2C%22id%22%3A%22default-romain%22%2C%22name%22%3A%22default-romain%22%7D%2C%22database%22%3A%22default%22%2C%22currentQueryTab%22%3A%22queryHistory%22%2C%22pinnedContextTabs%22%3A%5B%5D%2C%22loadingQueries%22%3Afalse%2C%22queriesHasErrors%22%3Afalse%2C%22queriesCurrentPage%22%3A1%2C%22queriesTotalPages%22%3A1%2C%22queriesFilter%22%3A%22%22%2C%22queriesFilterVisible%22%3Afalse%2C%22statementType%22%3A%22text%22%2C%22statementTypes%22%3A%5B%22text%22%2C%22file%22%5D%2C%22statementPath%22%3A%22%22%2C%22externalStatementLoaded%22%3Afalse%2C%22associatedDocumentLoading%22%3Atrue%2C%22associatedDocumentUuid%22%3Anull%2C%22statement_raw%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22statementsList%22%3A%5B%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%5D%2C%22aceSize%22%3A100%2C%22status%22%3A%22running%22%2C%22statusForButtons%22%3A%22executing%22%2C%22properties%22%3A%7B%22files%22%3A%5B%5D%2C%22functions%22%3A%5B%5D%2C%22arguments%22%3A%5B%5D%2C%22settings%22%3A%5B%5D%7D%2C%22viewSettings%22%3A%7B%22placeHolder%22%3A%22Example%3A+SELECT+*+FROM+tablename%2C+or+press+CTRL+%2B+space%22%2C%22sqlDialect%22%3Atrue%7D%2C%22variables%22%3A%5B%5D%2C%22hasCurlyBracketParameters%22%3Atrue%2C%22variableNames%22%3A%5B%5D%2C%22variableValues%22%3A%7B%7D%2C%22statement%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22result%22%3A%7B%22id%22%3A%2206840534-9434-33b5-5eca-2cd08432ceb3%22%2C%22type%22%3A%22table%22%2C%22hasResultset%22%3Atrue%2C%22handle%22%3A%7B%22has_more_statements%22%3Afalse%2C%22statement_id%22%3A0%2C%22statements_count%22%3A1%2C%22previous_statement_hash%22%3A%22acb6478fcf28c31b5e76d49de7d77bbe46fe5e4f9436c16c0ca8ed5f%22%7D%2C%22meta%22%3A%5B%5D%2C%22rows%22%3Anull%2C%22hasMore%22%3Afalse%2C%22statement_id%22%3A0%2C%22statement_range%22%3A%7B%22start%22%3A%7B%22row%22%3A0%2C%22column%22%3A0%7D%2C%22end%22%3A%7B%22row%22%3A0%2C%22column%22%3A0%7D%7D%2C%22statements_count%22%3A1%2C%22previous_statement_hash%22%3Anull%2C%22metaFilter%22%3A%7B%22query%22%3A%22%22%2C%22facets%22%3A%7B%7D%2C%22text%22%3A%5B%5D%7D%2C%22isMetaFilterVisible%22%3Afalse%2C%22filteredMetaChecked%22%3Atrue%2C%22filteredMeta%22%3A%5B%5D%2C%22fetchedOnce%22%3Afalse%2C%22startTime%22%3A%222018-06-12T16%3A15%3A01.951Z%22%2C%22endTime%22%3A%222018-06-12T16%3A15%3A01.951Z%22%2C%22executionTime%22%3A0%2C%22data%22%3A%5B%5D%2C%22explanation%22%3A%22%22%2C%22logs%22%3A%22%22%2C%22logLines%22%3A0%2C%22hasSomeResults%22%3Afalse%7D%2C%22showGrid%22%3Atrue%2C%22showChart%22%3Afalse%2C%22showLogs%22%3Atrue%2C%22progress%22%3A0%2C%22jobs%22%3A%5B%5D%2C%22isLoading%22%3Afalse%2C%22resultsKlass%22%3A%22results+impala%22%2C%22errorsKlass%22%3A%22results+impala+alert+alert-error%22%2C%22is_redacted%22%3Afalse%2C%22chartType%22%3A%22bars%22%2C%22chartSorting%22%3A%22none%22%2C%22chartScatterGroup%22%3Anull%2C%22chartScatterSize%22%3Anull%2C%22chartScope%22%3A%22world%22%2C%22chartTimelineType%22%3A%22bar%22%2C%22chartLimits%22%3A%5B5%2C10%2C25%2C50%2C100%5D%2C%22chartLimit%22%3Anull%2C%22chartX%22%3Anull%2C%22chartXPivot%22%3Anull%2C%22chartYSingle%22%3Anull%2C%22chartYMulti%22%3A%5B%5D%2C%22chartData%22%3A%5B%5D%2C%22chartMapType%22%3A%22marker%22%2C%22chartMapLabel%22%3Anull%2C%22chartMapHeat%22%3Anull%2C%22hideStacked%22%3Atrue%2C%22hasDataForChart%22%3Afalse%2C%22previousChartOptions%22%3A%7B%22chartLimit%22%3Anull%2C%22chartX%22%3Anull%2C%22chartXPivot%22%3Anull%2C%22chartYSingle%22%3Anull%2C%22chartMapType%22%3A%22marker%22%2C%22chartMapLabel%22%3Anull%2C%22chartMapHeat%22%3Anull%2C%22chartYMulti%22%3A%5B%5D%2C%22chartScope%22%3A%22world%22%2C%22chartTimelineType%22%3A%22bar%22%2C%22chartSorting%22%3A%22none%22%2C%22chartScatterGroup%22%3Anull%2C%22chartScatterSize%22%3Anull%7D%2C%22isResultSettingsVisible%22%3Afalse%2C%22settingsVisible%22%3Afalse%2C%22checkStatusTimeout%22%3Anull%2C%22topRisk%22%3Anull%2C%22suggestion%22%3A%22%22%2C%22hasSuggestion%22%3Anull%2C%22compatibilityCheckRunning%22%3Afalse%2C%22compatibilitySourcePlatform%22%3A%22impala%22%2C%22compatibilitySourcePlatforms%22%3A%5B%7B%22name%22%3A%22Teradata%22%2C%22value%22%3A%22teradata%22%7D%2C%7B%22name%22%3A%22Oracle%22%2C%22value%22%3A%22oracle%22%7D%2C%7B%22name%22%3A%22Netezza%22%2C%22value%22%3A%22netezza%22%7D%2C%7B%22name%22%3A%22Impala%22%2C%22value%22%3A%22impala%22%7D%2C%7B%22name%22%3A%22impala%22%2C%22value%22%3A%22impala%22%7D%2C%7B%22name%22%3A%22DB2%22%2C%22value%22%3A%22db2%22%7D%2C%7B%22name%22%3A%22Greenplum%22%2C%22value%22%3A%22greenplum%22%7D%2C%7B%22name%22%3A%22MySQL%22%2C%22value%22%3A%22mysql%22%7D%2C%7B%22name%22%3A%22PostgreSQL%22%2C%22value%22%3A%22postgresql%22%7D%2C%7B%22name%22%3A%22Informix%22%2C%22value%22%3A%22informix%22%7D%2C%7B%22name%22%3A%22SQL+Server%22%2C%22value%22%3A%22sqlserver%22%7D%2C%7B%22name%22%3A%22Sybase%22%2C%22value%22%3A%22sybase%22%7D%2C%7B%22name%22%3A%22Access%22%2C%22value%22%3A%22access%22%7D%2C%7B%22name%22%3A%22Firebird%22%2C%22value%22%3A%22firebird%22%7D%2C%7B%22name%22%3A%22ANSISQL%22%2C%22value%22%3A%22ansisql%22%7D%2C%7B%22name%22%3A%22Generic%22%2C%22value%22%3A%22generic%22%7D%5D%2C%22compatibilityTargetPlatform%22%3A%22impala%22%2C%22compatibilityTargetPlatforms%22%3A%5B%7B%22name%22%3A%22Impala%22%2C%22value%22%3A%22impala%22%7D%2C%7B%22name%22%3A%22impala%22%2C%22value%22%3A%22impala%22%7D%5D%2C%22showOptimizer%22%3Atrue%2C%22delayedStatement%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22wasBatchExecuted%22%3Afalse%2C%22isReady%22%3Atrue%2C%22lastExecuted%22%3A1528820101947%2C%22lastAceSelectionRowOffset%22%3A0%2C%22executingBlockingOperation%22%3Anull%2C%22showLongOperationWarning%22%3Afalse%2C%22formatEnabled%22%3Atrue%2C%22isFetchingData%22%3Afalse%2C%22isCanceling%22%3Afalse%2C%22aceAutoExpand%22%3Afalse%7D%5D%2C%22selectedSnippet%22%3A%22impala%22%2C%22creatingSessionLocks%22%3A%5B%5D%2C%22sessions%22%3A%5B%7B%22type%22%3A%22impala%22%2C%22properties%22%3A%5B%7B%22multiple%22%3Atrue%2C%22defaultValue%22%3A%5B%5D%2C%22value%22%3A%5B%5D%2C%22nice_name%22%3A%22Files%22%2C%22key%22%3A%22files%22%2C%22help_text%22%3A%22Add+one+or+more+files%2C+jars%2C+or+arcimpalas+to+the+list+of+resources.%22%2C%22type%22%3A%22hdfs-files%22%7D%2C%7B%22multiple%22%3Atrue%2C%22defaultValue%22%3A%5B%5D%2C%22value%22%3A%5B%5D%2C%22nice_name%22%3A%22Functions%22%2C%22key%22%3A%22functions%22%2C%22help_text%22%3A%22Add+one+or+more+registered+UDFs+(requires+function+name+and+fully-qualified+class+name).%22%2C%22type%22%3A%22functions%22%7D%2C%7B%22nice_name%22%3A%22Settings%22%2C%22multiple%22%3Atrue%2C%22key%22%3A%22settings%22%2C%22help_text%22%3A%22impala+and+Hadoop+configuration+properties.%22%2C%22defaultValue%22%3A%5B%5D%2C%22type%22%3A%22settings%22%2C%22options%22%3A%5B%22impala.map.aggr%22%2C%22impala.exec.compress.output%22%2C%22impala.exec.parallel%22%2C%22impala.execution.engine%22%2C%22mapreduce.job.queuename%22%5D%2C%22value%22%3A%5B%5D%7D%5D%2C%22reuse_session%22%3Atrue%2C%22id%22%3A6865%2C%22session_id%22%3A%22714fb09b96ba3368%3A4d02ec93d7ffbfb6%22%7D%5D%2C%22directoryUuid%22%3A%22%22%2C%22dependentsCoordinator%22%3A%5B%5D%2C%22historyFilter%22%3A%22%22%2C%22historyFilterVisible%22%3Afalse%2C%22loadingHistory%22%3Afalse%2C%22historyInitialHeight%22%3A1679%2C%22forceHistoryInitialHeight%22%3Atrue%2C%22historyCurrentPage%22%3A1%2C%22historyTotalPages%22%3A3%2C%22schedulerViewModel%22%3Anull%2C%22schedulerViewModelIsLoaded%22%3Afalse%2C%22isBatchable%22%3Atrue%2C%22isExecutingAll%22%3Afalse%2C%22executingAllIndex%22%3A0%2C%22retryModalConfirm%22%3Anull%2C%22retryModalCancel%22%3Anull%2C%22unloaded%22%3Afalse%2C%22updateHistoryFailed%22%3Afalse%2C%22viewSchedulerId%22%3A%22%22%2C%22loadingScheduler%22%3Afalse%7D&snippet=%7B%22id%22%3A%22dd5755a3-e8db-82d9-4f98-9f4fb5a99a06%22%2C%22type%22%3A%22impala%22%2C%22status%22%3A%22running%22%2C%22statementType%22%3A%22text%22%2C%22statement%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22aceCursorPosition%22%3A%7B%22column%22%3A33%2C%22row%22%3A0%7D%2C%22statementPath%22%3A%22%22%2C%22associatedDocumentUuid%22%3Anull%2C%22properties%22%3A%7B%22files%22%3A%5B%5D%2C%22functions%22%3A%5B%5D%2C%22arguments%22%3A%5B%5D%2C%22settings%22%3A%5B%5D%7D%2C%22result%22%3A%7B%22id%22%3A%2206840534-9434-33b5-5eca-2cd08432ceb3%22%2C%22type%22%3A%22table%22%2C%22handle%22%3A%7B%22has_more_statements%22%3Afalse%2C%22statement_id%22%3A0%2C%22statements_count%22%3A1%2C%22previous_statement_hash%22%3A%22acb6478fcf28c31b5e76d49de7d77bbe46fe5e4f9436c16c0ca8ed5f%22%7D%7D%2C%22database%22%3A%22default%22%2C%22compute%22%3A%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22default-romain%22%2C%22id%22%3A%22default-romain%22%2C%22name%22%3A%22default-romain%22%7D%2C%22wasBatchExecuted%22%3Afalse%7D", "params": [ { "name": "notebook", "value": "%7B%22uuid%22%3A%22f2b8a233-c34c-44b8-a8a1-0e6123996216%22%2C%22name%22%3A%22%22%2C%22description%22%3A%22%22%2C%22type%22%3A%22query-impala%22%2C%22initialType%22%3A%22impala%22%2C%22coordinatorUuid%22%3Anull%2C%22isHistory%22%3Atrue%2C%22isManaged%22%3Afalse%2C%22parentSavedQueryUuid%22%3Anull%2C%22isSaved%22%3Afalse%2C%22onSuccessUrl%22%3Anull%2C%22pubSubUrl%22%3Anull%2C%22isPresentationModeDefault%22%3Afalse%2C%22isPresentationMode%22%3Afalse%2C%22isPresentationModeInitialized%22%3Atrue%2C%22presentationSnippets%22%3A%7B%7D%2C%22isHidingCode%22%3Afalse%2C%22snippets%22%3A%5B%7B%22id%22%3A%22dd5755a3-e8db-82d9-4f98-9f4fb5a99a06%22%2C%22name%22%3A%22%22%2C%22type%22%3A%22impala%22%2C%22isBatchable%22%3Atrue%2C%22aceCursorPosition%22%3A%7B%22column%22%3A33%2C%22row%22%3A0%7D%2C%22errors%22%3A%5B%5D%2C%22aceErrorsHolder%22%3A%5B%5D%2C%22aceWarningsHolder%22%3A%5B%5D%2C%22aceErrors%22%3A%5B%5D%2C%22aceWarnings%22%3A%5B%5D%2C%22editorMode%22%3Atrue%2C%22dbSelectionVisible%22%3Afalse%2C%22isSqlDialect%22%3Atrue%2C%22namespaceRefreshEnabled%22%3Afalse%2C%22availableNamespaces%22%3A%5B%5D%2C%22availableComputes%22%3A%5B%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22default-romain%22%2C%22id%22%3A%22default-romain%22%2C%22name%22%3A%22default-romain%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22compute1%22%2C%22id%22%3A%22compute1%22%2C%22name%22%3A%22compute1%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22compute2%22%2C%22id%22%3A%22compute2%22%2C%22name%22%3A%22compute2%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22altus%22%2C%22namespace%22%3A%22Altus%22%2C%22id%22%3A%22Altus%22%2C%22name%22%3A%22Altus%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22storage1%22%2C%22id%22%3A%22storage1%22%2C%22name%22%3A%22storage1%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22storage2%22%2C%22id%22%3A%22storage2%22%2C%22name%22%3A%22storage2%22%7D%2C%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22Default%22%2C%22id%22%3A%22Default%22%2C%22name%22%3A%22default%22%7D%5D%2C%22compute%22%3A%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22default-romain%22%2C%22id%22%3A%22default-romain%22%2C%22name%22%3A%22default-romain%22%7D%2C%22database%22%3A%22default%22%2C%22currentQueryTab%22%3A%22queryHistory%22%2C%22pinnedContextTabs%22%3A%5B%5D%2C%22loadingQueries%22%3Afalse%2C%22queriesHasErrors%22%3Afalse%2C%22queriesCurrentPage%22%3A1%2C%22queriesTotalPages%22%3A1%2C%22queriesFilter%22%3A%22%22%2C%22queriesFilterVisible%22%3Afalse%2C%22statementType%22%3A%22text%22%2C%22statementTypes%22%3A%5B%22text%22%2C%22file%22%5D%2C%22statementPath%22%3A%22%22%2C%22externalStatementLoaded%22%3Afalse%2C%22associatedDocumentLoading%22%3Atrue%2C%22associatedDocumentUuid%22%3Anull%2C%22statement_raw%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22statementsList%22%3A%5B%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%5D%2C%22aceSize%22%3A100%2C%22status%22%3A%22running%22%2C%22statusForButtons%22%3A%22executing%22%2C%22properties%22%3A%7B%22files%22%3A%5B%5D%2C%22functions%22%3A%5B%5D%2C%22arguments%22%3A%5B%5D%2C%22settings%22%3A%5B%5D%7D%2C%22viewSettings%22%3A%7B%22placeHolder%22%3A%22Example%3A+SELECT+*+FROM+tablename%2C+or+press+CTRL+%2B+space%22%2C%22sqlDialect%22%3Atrue%7D%2C%22variables%22%3A%5B%5D%2C%22hasCurlyBracketParameters%22%3Atrue%2C%22variableNames%22%3A%5B%5D%2C%22variableValues%22%3A%7B%7D%2C%22statement%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22result%22%3A%7B%22id%22%3A%2206840534-9434-33b5-5eca-2cd08432ceb3%22%2C%22type%22%3A%22table%22%2C%22hasResultset%22%3Atrue%2C%22handle%22%3A%7B%22has_more_statements%22%3Afalse%2C%22statement_id%22%3A0%2C%22statements_count%22%3A1%2C%22previous_statement_hash%22%3A%22acb6478fcf28c31b5e76d49de7d77bbe46fe5e4f9436c16c0ca8ed5f%22%7D%2C%22meta%22%3A%5B%5D%2C%22rows%22%3Anull%2C%22hasMore%22%3Afalse%2C%22statement_id%22%3A0%2C%22statement_range%22%3A%7B%22start%22%3A%7B%22row%22%3A0%2C%22column%22%3A0%7D%2C%22end%22%3A%7B%22row%22%3A0%2C%22column%22%3A0%7D%7D%2C%22statements_count%22%3A1%2C%22previous_statement_hash%22%3Anull%2C%22metaFilter%22%3A%7B%22query%22%3A%22%22%2C%22facets%22%3A%7B%7D%2C%22text%22%3A%5B%5D%7D%2C%22isMetaFilterVisible%22%3Afalse%2C%22filteredMetaChecked%22%3Atrue%2C%22filteredMeta%22%3A%5B%5D%2C%22fetchedOnce%22%3Afalse%2C%22startTime%22%3A%222018-06-12T16%3A15%3A01.951Z%22%2C%22endTime%22%3A%222018-06-12T16%3A15%3A01.951Z%22%2C%22executionTime%22%3A0%2C%22data%22%3A%5B%5D%2C%22explanation%22%3A%22%22%2C%22logs%22%3A%22%22%2C%22logLines%22%3A0%2C%22hasSomeResults%22%3Afalse%7D%2C%22showGrid%22%3Atrue%2C%22showChart%22%3Afalse%2C%22showLogs%22%3Atrue%2C%22progress%22%3A0%2C%22jobs%22%3A%5B%5D%2C%22isLoading%22%3Afalse%2C%22resultsKlass%22%3A%22results+impala%22%2C%22errorsKlass%22%3A%22results+impala+alert+alert-error%22%2C%22is_redacted%22%3Afalse%2C%22chartType%22%3A%22bars%22%2C%22chartSorting%22%3A%22none%22%2C%22chartScatterGroup%22%3Anull%2C%22chartScatterSize%22%3Anull%2C%22chartScope%22%3A%22world%22%2C%22chartTimelineType%22%3A%22bar%22%2C%22chartLimits%22%3A%5B5%2C10%2C25%2C50%2C100%5D%2C%22chartLimit%22%3Anull%2C%22chartX%22%3Anull%2C%22chartXPivot%22%3Anull%2C%22chartYSingle%22%3Anull%2C%22chartYMulti%22%3A%5B%5D%2C%22chartData%22%3A%5B%5D%2C%22chartMapType%22%3A%22marker%22%2C%22chartMapLabel%22%3Anull%2C%22chartMapHeat%22%3Anull%2C%22hideStacked%22%3Atrue%2C%22hasDataForChart%22%3Afalse%2C%22previousChartOptions%22%3A%7B%22chartLimit%22%3Anull%2C%22chartX%22%3Anull%2C%22chartXPivot%22%3Anull%2C%22chartYSingle%22%3Anull%2C%22chartMapType%22%3A%22marker%22%2C%22chartMapLabel%22%3Anull%2C%22chartMapHeat%22%3Anull%2C%22chartYMulti%22%3A%5B%5D%2C%22chartScope%22%3A%22world%22%2C%22chartTimelineType%22%3A%22bar%22%2C%22chartSorting%22%3A%22none%22%2C%22chartScatterGroup%22%3Anull%2C%22chartScatterSize%22%3Anull%7D%2C%22isResultSettingsVisible%22%3Afalse%2C%22settingsVisible%22%3Afalse%2C%22checkStatusTimeout%22%3Anull%2C%22topRisk%22%3Anull%2C%22suggestion%22%3A%22%22%2C%22hasSuggestion%22%3Anull%2C%22compatibilityCheckRunning%22%3Afalse%2C%22compatibilitySourcePlatform%22%3A%22impala%22%2C%22compatibilitySourcePlatforms%22%3A%5B%7B%22name%22%3A%22Teradata%22%2C%22value%22%3A%22teradata%22%7D%2C%7B%22name%22%3A%22Oracle%22%2C%22value%22%3A%22oracle%22%7D%2C%7B%22name%22%3A%22Netezza%22%2C%22value%22%3A%22netezza%22%7D%2C%7B%22name%22%3A%22Impala%22%2C%22value%22%3A%22impala%22%7D%2C%7B%22name%22%3A%22impala%22%2C%22value%22%3A%22impala%22%7D%2C%7B%22name%22%3A%22DB2%22%2C%22value%22%3A%22db2%22%7D%2C%7B%22name%22%3A%22Greenplum%22%2C%22value%22%3A%22greenplum%22%7D%2C%7B%22name%22%3A%22MySQL%22%2C%22value%22%3A%22mysql%22%7D%2C%7B%22name%22%3A%22PostgreSQL%22%2C%22value%22%3A%22postgresql%22%7D%2C%7B%22name%22%3A%22Informix%22%2C%22value%22%3A%22informix%22%7D%2C%7B%22name%22%3A%22SQL+Server%22%2C%22value%22%3A%22sqlserver%22%7D%2C%7B%22name%22%3A%22Sybase%22%2C%22value%22%3A%22sybase%22%7D%2C%7B%22name%22%3A%22Access%22%2C%22value%22%3A%22access%22%7D%2C%7B%22name%22%3A%22Firebird%22%2C%22value%22%3A%22firebird%22%7D%2C%7B%22name%22%3A%22ANSISQL%22%2C%22value%22%3A%22ansisql%22%7D%2C%7B%22name%22%3A%22Generic%22%2C%22value%22%3A%22generic%22%7D%5D%2C%22compatibilityTargetPlatform%22%3A%22impala%22%2C%22compatibilityTargetPlatforms%22%3A%5B%7B%22name%22%3A%22Impala%22%2C%22value%22%3A%22impala%22%7D%2C%7B%22name%22%3A%22impala%22%2C%22value%22%3A%22impala%22%7D%5D%2C%22showOptimizer%22%3Atrue%2C%22delayedStatement%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22wasBatchExecuted%22%3Afalse%2C%22isReady%22%3Atrue%2C%22lastExecuted%22%3A1528820101947%2C%22lastAceSelectionRowOffset%22%3A0%2C%22executingBlockingOperation%22%3Anull%2C%22showLongOperationWarning%22%3Afalse%2C%22formatEnabled%22%3Atrue%2C%22isFetchingData%22%3Afalse%2C%22isCanceling%22%3Afalse%2C%22aceAutoExpand%22%3Afalse%7D%5D%2C%22selectedSnippet%22%3A%22impala%22%2C%22creatingSessionLocks%22%3A%5B%5D%2C%22sessions%22%3A%5B%7B%22type%22%3A%22impala%22%2C%22properties%22%3A%5B%7B%22multiple%22%3Atrue%2C%22defaultValue%22%3A%5B%5D%2C%22value%22%3A%5B%5D%2C%22nice_name%22%3A%22Files%22%2C%22key%22%3A%22files%22%2C%22help_text%22%3A%22Add+one+or+more+files%2C+jars%2C+or+arcimpalas+to+the+list+of+resources.%22%2C%22type%22%3A%22hdfs-files%22%7D%2C%7B%22multiple%22%3Atrue%2C%22defaultValue%22%3A%5B%5D%2C%22value%22%3A%5B%5D%2C%22nice_name%22%3A%22Functions%22%2C%22key%22%3A%22functions%22%2C%22help_text%22%3A%22Add+one+or+more+registered+UDFs+(requires+function+name+and+fully-qualified+class+name).%22%2C%22type%22%3A%22functions%22%7D%2C%7B%22nice_name%22%3A%22Settings%22%2C%22multiple%22%3Atrue%2C%22key%22%3A%22settings%22%2C%22help_text%22%3A%22impala+and+Hadoop+configuration+properties.%22%2C%22defaultValue%22%3A%5B%5D%2C%22type%22%3A%22settings%22%2C%22options%22%3A%5B%22impala.map.aggr%22%2C%22impala.exec.compress.output%22%2C%22impala.exec.parallel%22%2C%22impala.execution.engine%22%2C%22mapreduce.job.queuename%22%5D%2C%22value%22%3A%5B%5D%7D%5D%2C%22reuse_session%22%3Atrue%2C%22id%22%3A6865%2C%22session_id%22%3A%22714fb09b96ba3368%3A4d02ec93d7ffbfb6%22%7D%5D%2C%22directoryUuid%22%3A%22%22%2C%22dependentsCoordinator%22%3A%5B%5D%2C%22historyFilter%22%3A%22%22%2C%22historyFilterVisible%22%3Afalse%2C%22loadingHistory%22%3Afalse%2C%22historyInitialHeight%22%3A1679%2C%22forceHistoryInitialHeight%22%3Atrue%2C%22historyCurrentPage%22%3A1%2C%22historyTotalPages%22%3A3%2C%22schedulerViewModel%22%3Anull%2C%22schedulerViewModelIsLoaded%22%3Afalse%2C%22isBatchable%22%3Atrue%2C%22isExecutingAll%22%3Afalse%2C%22executingAllIndex%22%3A0%2C%22retryModalConfirm%22%3Anull%2C%22retryModalCancel%22%3Anull%2C%22unloaded%22%3Afalse%2C%22updateHistoryFailed%22%3Afalse%2C%22viewSchedulerId%22%3A%22%22%2C%22loadingScheduler%22%3Afalse%7D" }, { "name": "snippet", "value": "%7B%22id%22%3A%22dd5755a3-e8db-82d9-4f98-9f4fb5a99a06%22%2C%22type%22%3A%22impala%22%2C%22status%22%3A%22running%22%2C%22statementType%22%3A%22text%22%2C%22statement%22%3A%22SELECT+*+FROM+web_logs+LIMIT+100%3B%22%2C%22aceCursorPosition%22%3A%7B%22column%22%3A33%2C%22row%22%3A0%7D%2C%22statementPath%22%3A%22%22%2C%22associatedDocumentUuid%22%3Anull%2C%22properties%22%3A%7B%22files%22%3A%5B%5D%2C%22functions%22%3A%5B%5D%2C%22arguments%22%3A%5B%5D%2C%22settings%22%3A%5B%5D%7D%2C%22result%22%3A%7B%22id%22%3A%2206840534-9434-33b5-5eca-2cd08432ceb3%22%2C%22type%22%3A%22table%22%2C%22handle%22%3A%7B%22has_more_statements%22%3Afalse%2C%22statement_id%22%3A0%2C%22statements_count%22%3A1%2C%22previous_statement_hash%22%3A%22acb6478fcf28c31b5e76d49de7d77bbe46fe5e4f9436c16c0ca8ed5f%22%7D%7D%2C%22database%22%3A%22default%22%2C%22compute%22%3A%7B%22interface%22%3A%22impala%22%2C%22type%22%3A%22direct%22%2C%22namespace%22%3A%22default-romain%22%2C%22id%22%3A%22default-romain%22%2C%22name%22%3A%22default-romain%22%7D%2C%22wasBatchExecuted%22%3Afalse%7D" } ] } }''' payload = payload.replace('SELECT+*+FROM+web_logs+LIMIT+100', urllib.quote_plus(query.replace('\n', ' '))) resp = self.api.submit_hue_query(self.cluster_crn, payload) resp_payload = json.loads(resp['payload']) if 'handle' in resp_payload: return resp_payload['handle'] else: raise QueryError(resp_payload.get('message')) def do_check_status(self, handle): notebook = { "type": "impala", "name": "query", "isSaved": False, "sessions": [], "snippets": [{ "id": "1234", "type": "impala", "statement_raw": "SHOW DATABASES", "result": { "handle": {} } }] } snippet = { "id": "1234", "type": "impala", "statement": "SHOW DATABASES", "status": "running", "result": { 'handle': { "log_context": None, "statements_count": 1, "end": { "column": 13, "row": 0 }, "statement_id": 0, "has_more_statements": False, "start": { "column": 0, "row": 0 }, "secret": "3h9WBnLbTUYAAAAAPQjxlQ==\n", "has_result_set": True, "session_guid": "qcrpEBmCTGacxfhM+CxbkQ==\n", "statement": "SHOW DATABASES", "operation_type": 0, "modified_row_count": None, "guid": "3h9WBnLbTUYAAAAAPQjxlQ==\n", "previous_statement_hash": "5b1f14102d749be7b41da376bcdbb64f993ce00bc46e3aab0b8008c4" } }, "properties": {} } snippet['result']['handle'] = handle notebook_payload = urllib.quote(json.dumps(notebook)) snippet_payload = urllib.quote(json.dumps(snippet)) payload = ''' { "method": "POST", "url": "http://127.0.0.1:8000/notebook/api/check_status", "httpVersion": "HTTP/1.1", "headers": [ { "name": "Accept-Encoding", "value": "gzip, deflate, br" }, { "name": "Content-Type", "value": "application/x-www-form-urlencoded; charset=UTF-8" }, { "name": "Accept", "value": "*/*" }, { "name": "X-Requested-With", "value": "XMLHttpRequest" }, { "name": "Connection", "value": "keep-alive" } ], "queryString": [], "cookies": [ ], "postData": { "mimeType": "application/x-www-form-urlencoded; charset=UTF-8", "text": "notebook=%(notebook)s&snippet=%(snippet)s", "params": [ { "name": "notebook", "value": "%(notebook)s" }, { "name": "snippet", "value": "%(snippet)s" } ] } }''' % { 'notebook': notebook_payload, 'snippet': snippet_payload } resp = self.api.submit_hue_query(self.cluster_crn, payload) resp_payload = json.loads(resp['payload']) if 'query_status' in resp_payload: return resp_payload['query_status'] else: return resp_payload def do_fetch_result(self, handle): notebook = { "type": "impala", "name": "query", "isSaved": False, "sessions": [], "snippets": [{ "id": "1234", "type": "impala", "statement_raw": "SHOW DATABASES", "result": { "handle": {} } }] } snippet = { "id": "1234", "type": "impala", "statement": "SHOW DATABASES", "status": "running", "result": { 'handle': { "log_context": None, "statements_count": 1, "end": { "column": 13, "row": 0 }, "statement_id": 0, "has_more_statements": False, "start": { "column": 0, "row": 0 }, "secret": "3h9WBnLbTUYAAAAAPQjxlQ==\n", "has_result_set": True, "session_guid": "qcrpEBmCTGacxfhM+CxbkQ==\n", "statement": "SHOW DATABASES", "operation_type": 0, "modified_row_count": None, "guid": "3h9WBnLbTUYAAAAAPQjxlQ==\n", "previous_statement_hash": "5b1f14102d749be7b41da376bcdbb64f993ce00bc46e3aab0b8008c4" } }, "properties": {} } rows = 100 start_over = True snippet['result']['handle'] = handle notebook_payload = urllib.quote(json.dumps(notebook)) snippet_payload = urllib.quote(json.dumps(snippet)) rows_payload = urllib.quote(json.dumps(rows)) start_over_payload = urllib.quote(json.dumps(start_over)) payload = ''' { "method": "POST", "url": "http://127.0.0.1:8000/notebook/api/fetch_result_data", "httpVersion": "HTTP/1.1", "headers": [ { "name": "Accept-Encoding", "value": "gzip, deflate, br" }, { "name": "Content-Type", "value": "application/x-www-form-urlencoded; charset=UTF-8" }, { "name": "Accept", "value": "*/*" }, { "name": "X-Requested-With", "value": "XMLHttpRequest" }, { "name": "Connection", "value": "keep-alive" } ], "queryString": [], "cookies": [ ], "postData": { "mimeType": "application/x-www-form-urlencoded; charset=UTF-8", "text": "notebook=%(notebook)s&snippet=%(snippet)s&rows=%(rows)s&startOver=%(start_over)s", "params": [ { "name": "notebook", "value": "%(notebook)s" }, { "name": "snippet", "value": "%(snippet)s" }, { "name": "rows", "value": %(rows)s }, { "name": "startOver", "value": "%(start_over)s" } ] } }''' % { 'notebook': notebook_payload, 'snippet': snippet_payload, 'rows': rows_payload, 'start_over': start_over_payload } resp = self.api.submit_hue_query(self.cluster_crn, payload) return json.loads(resp['payload'])['result']
def get_context_computes(request, interface): response = {} computes = [] clusters = get_clusters(request.user).values() if interface == 'hive' or interface == 'impala' or interface == 'oozie' or interface == 'report': computes.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'namespace': cluster['id'], 'interface': interface, 'type': cluster['type'] } for cluster in clusters if cluster.get('type') == 'direct']) if interface == 'impala' or interface == 'report': if [cluster for cluster in clusters if cluster['type'] == 'altus']: computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'namespace': cluster.get('namespaceCrn', cluster.get('crn')), 'type': 'altus-dw' } for cluster in AnalyticDbApi( request.user).list_clusters()['clusters'] if cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515']) if interface == 'oozie' or interface == 'spark2': if [cluster for cluster in clusters if cluster['type'] == 'altus']: computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'environmentType': cluster.get('environmentType'), 'serviceType': cluster.get('serviceType'), 'namespace': cluster.get('namespaceCrn'), 'type': 'altus-de' } for cluster in DataEngApi(request.user).list_clusters() ['clusters']]) # TODO if interface == 'spark2' keep only SPARK type if interface == 'jobs': for cluster in clusters: cluster = { 'id': cluster.get('id'), 'name': cluster.get('name'), 'status': 'CREATED', 'environmentType': cluster.get('type'), 'serviceType': cluster.get('interface'), 'namespace': '', 'type': cluster.get('type') } if cluster.get('type') == 'altus': cluster['name'] = 'Altus DE' cluster['type'] = 'altus-de' computes.append(cluster) cluster = cluster.copy() cluster['name'] = 'Altus Data Warehouse' cluster['type'] = 'altus-dw' computes.append(cluster) response[interface] = computes response['status'] = 0 return JsonResponse(response)