def _importer(request, prefill): source_type = request.GET.get('sourceType') or get_cluster_config( request.user)['default_sql_interpreter'] return render( 'importer.mako', request, { 'is_embeddable': request.GET.get('is_embeddable', False), 'fields_json': json.dumps({ 'solr': [field.name for field in FIELD_TYPES], 'hive': HIVE_TYPES, 'hivePrimitive': HIVE_PRIMITIVE_TYPES }), 'operators_json': json.dumps([operator.to_dict() for operator in OPERATORS]), 'file_types_json': json.dumps([ format_.format_info() for format_ in get_file_indexable_format_types() ]), 'default_field_type': json.dumps(Field().to_dict()), 'prefill': json.dumps(prefill), 'source_type': source_type })
def _get_db(user, source_type=None): if source_type is None: source_type = get_cluster_config(user)['default_sql_interpreter'] query_server = get_query_server_config( name=source_type if source_type != 'hive' else 'beeswax') return dbms.get(user, query_server)
def get_config(request): if request.POST.get(USER_PREFERENCE_CLUSTER): set_user_preferences(request.user, USER_PREFERENCE_CLUSTER, request.POST.get(USER_PREFERENCE_CLUSTER)) config = get_cluster_config(request.user) config['status'] = 0 return JsonResponse(config)
def get_config(request): config = get_cluster_config(request.user) config['clusters'] = list(get_clusters(request.user).values()) config['documents'] = { 'types': list(Document2.objects.documents(user=request.user).order_by().values_list('type', flat=True).distinct()) } config['status'] = 0 return JsonResponse(config)
def get_context_namespaces(request, interface): ''' Namespaces are node cluster contexts (e.g. Hive + Ranger) that can be queried by computes. ''' response = {} namespaces = [] clusters = list(get_clusters(request.user).values()) # Currently broken if not sent namespaces.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'status': 'CREATED', 'computes': [cluster] } for cluster in clusters if cluster.get('type') == 'direct' ]) if interface == 'hive' or interface == 'impala' or interface == 'report': if get_cluster_config(request.user)['has_computes']: # Note: attaching computes to namespaces might be done via the frontend in the future if interface == 'impala': if IS_K8S_ONLY.get(): adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] for _cluster in adb_clusters: # Add "fake" namespace if needed if not _cluster.get('namespaceCrn'): _cluster['namespaceCrn'] = _cluster['crn'] _cluster['id'] = _cluster['crn'] _cluster['namespaceName'] = _cluster['clusterName'] _cluster['name'] = _cluster['clusterName'] _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8S_ONLY.get() else '', else: adb_clusters = [] if IS_K8S_ONLY.get(): sdx_namespaces = [] else: sdx_namespaces = SdxApi(request.user).list_namespaces() # Adding "fake" namespace for cluster without one sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])]) namespaces.extend([{ 'id': namespace.get('crn', 'None'), 'name': namespace.get('namespaceName'), 'status': namespace.get('status'), 'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')] } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get() ]) response[interface] = namespaces response['status'] = 0 return JsonResponse(response)
def _get_db(user, source_type=None): if source_type is None: cluster_config = get_cluster_config(user) if FORCE_HS2_METADATA.get() and cluster_config['app_config'].get('editor') and 'hive' in cluster_config['app_config'].get('editor')['interpreter_names']: source_type = 'hive' else: source_type = cluster_config['default_sql_interpreter'] query_server = get_query_server_config(name=source_type if source_type != 'hive' else 'beeswax') return dbms.get(user, query_server)
def _make_select_statement_gist(host_domain, is_http_secure, user, channel_id, statement): default_dialect = get_cluster_config( rewrite_user(user))['main_button_action']['dialect'] gist_response = _gist_create(host_domain, is_http_secure, user, statement, default_dialect) msg = 'Here is the gist link\n {gist_link}'.format( gist_link=gist_response['link']) _send_message(channel_id, message=msg)
def _get_db(user, source_type=None, cluster=None): if source_type is None: cluster_config = get_cluster_config(user) if FORCE_HS2_METADATA.get() and cluster_config['app_config'].get('editor') and 'hive' in cluster_config['app_config'].get('editor')['interpreter_names']: source_type = 'hive' else: source_type = cluster_config['default_sql_interpreter'] name = source_type if source_type != 'hive' else 'beeswax' query_server = get_query_server_config(name=name, cluster=cluster) return dbms.get(user, query_server)
def detect_select_statement(host_domain, is_http_secure, channel_id, user_id, statement): slack_user = check_slack_user_permission(host_domain, user_id) user = get_user(channel_id, slack_user) default_dialect = get_cluster_config( rewrite_user(user))['default_sql_interpreter'] gist_response = _gist_create(host_domain, is_http_secure, user, statement, default_dialect) bot_message = ( 'Hi <@{user}>\n' 'Looks like you are copy/pasting SQL, instead now you can send Editor links which unfurls in a rich preview!\n' 'Here is the gist link\n {gist_link}').format( user=user_id, gist_link=gist_response['link']) _send_message(channel_id, bot_message)
def get_context_computes(request, interface): ''' Some clusters like Snowball can have multiple computes for a certain languages (Hive, Impala...). ''' response = {} computes = [] clusters = list(get_clusters(request.user).values()) if get_cluster_config(request.user)['has_computes']: # TODO: only based on interface selected? interpreter = get_interpreter(connector_type=interface, user=request.user) if interpreter['dialect'] == 'impala': # dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] dw_clusters = [ {'crn': 'c1', 'clusterName': 'c1', 'status': 'created', 'options': {'server_host': 'c1.gethue.com', 'server_port': 10000}}, {'crn': 'c2', 'clusterName': 'c2', 'status': 'created', 'options': {'server_host': 'c2.gethue.com', 'server_port': 10000}}, ] computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'namespace': cluster.get('namespaceCrn', cluster.get('crn')), 'type': interpreter['dialect'], 'options': cluster['options'], } for cluster in dw_clusters] ) else: # Currently broken if not sent computes.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'namespace': cluster['id'], 'interface': interface, 'type': cluster['type'], 'options': {} } for cluster in clusters if cluster.get('type') == 'direct' ]) response[interface] = computes response['status'] = 0 return JsonResponse(response)
def get_api(request, snippet): from notebook.connectors.oozie_batch import OozieApi if snippet.get('wasBatchExecuted') and not TASK_SERVER.ENABLED.get(): return OozieApi(user=request.user, request=request) if snippet.get('type') == 'report': snippet['type'] = 'impala' patch_snippet_for_connector(snippet) connector_name = snippet['type'] if has_connectors() and snippet.get('type') == 'hello' and is_admin( request.user): interpreter = snippet.get('interpreter') else: interpreter = get_interpreter(connector_type=connector_name, user=request.user) interface = interpreter['interface'] if get_cluster_config(request.user)['has_computes']: compute = json.loads(request.POST.get( 'cluster', '""')) # Via Catalog autocomplete API or Notebook create sessions. if compute == '""' or compute == 'undefined': compute = None if not compute and snippet.get('compute'): # Via notebook.ko.js interpreter['compute'] = snippet['compute'] LOG.debug('Selected interpreter %s interface=%s compute=%s' % (interpreter['type'], interface, interpreter.get('compute') and interpreter['compute']['name'])) if interface == 'hiveserver2' or interface == 'hms': from notebook.connectors.hiveserver2 import HS2Api return HS2Api(user=request.user, request=request, interpreter=interpreter) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': from notebook.connectors.spark_shell import SparkApi return SparkApi(request.user, interpreter=interpreter) elif interface == 'livy-batch': from notebook.connectors.spark_batch import SparkBatchApi return SparkBatchApi(request.user, interpreter=interpreter) elif interface == 'text' or interface == 'markdown': from notebook.connectors.text import TextApi return TextApi(request.user) elif interface == 'rdbms': from notebook.connectors.rdbms import RdbmsApi return RdbmsApi(request.user, interpreter=snippet['type'], query_server=snippet.get('query_server')) elif interface == 'jdbc': if interpreter['options'] and interpreter['options'].get( 'url', '').find('teradata') >= 0: from notebook.connectors.jdbc_teradata import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) if interpreter['options'] and interpreter['options'].get( 'url', '').find('awsathena') >= 0: from notebook.connectors.jdbc_athena import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('presto') >= 0: from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('clickhouse') >= 0: from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse return JdbcApiClickhouse(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('vertica') >= 0: from notebook.connectors.jdbc_vertica import JdbcApiVertica return JdbcApiVertica(request.user, interpreter=interpreter) else: from notebook.connectors.jdbc import JdbcApi return JdbcApi(request.user, interpreter=interpreter) elif interface == 'teradata': from notebook.connectors.jdbc_teradata import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) elif interface == 'athena': from notebook.connectors.jdbc_athena import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interface == 'presto': from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interface == 'sqlalchemy': from notebook.connectors.sql_alchemy import SqlAlchemyApi return SqlAlchemyApi(request.user, interpreter=interpreter) elif interface == 'solr': from notebook.connectors.solr import SolrApi return SolrApi(request.user, interpreter=interpreter) elif interface == 'hbase': from notebook.connectors.hbase import HBaseApi return HBaseApi(request.user) elif interface == 'ksql': from notebook.connectors.ksql import KSqlApi return KSqlApi(request.user, interpreter=interpreter) elif interface == 'flink': from notebook.connectors.flink_sql import FlinkSqlApi return FlinkSqlApi(request.user, interpreter=interpreter) elif interface == 'kafka': from notebook.connectors.kafka import KafkaApi return KafkaApi(request.user) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException( _('Notebook connector interface not recognized: %s') % interface)
def get_config(request): config = get_cluster_config(request.user) config['status'] = 0 return JsonResponse(config)
def get_config(request): config = get_cluster_config(request.user) config['clusters'] = list(get_clusters(request.user).values()) config['status'] = 0 return JsonResponse(config)
def _submit_coordinator(request, coordinator, mapping): try: wf = coordinator.workflow if IS_MULTICLUSTER_ONLY.get() and get_cluster_config( request.user)['has_computes']: mapping['auto-cluster'] = { u'additionalClusterResourceTags': [], u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE', #'u'NONE', u'cdhVersion': u'CDH514', u'clouderaManagerPassword': u'guest', u'clouderaManagerUsername': u'guest', u'clusterName': u'analytics4', # Add time variable u'computeWorkersConfiguration': { u'bidUSDPerHr': 0, u'groupSize': 0, u'useSpot': False }, u'environmentName': u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946', u'instanceBootstrapScript': u'', u'instanceType': u'm4.xlarge', u'jobSubmissionGroupName': u'', u'jobs': [ { u'failureAction': u'INTERRUPT_JOB_QUEUE', u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51', u'sparkJob': { u'applicationArguments': ['5'], u'jars': [ u's3a://datawarehouse-customer360/ETL/spark-examples.jar' ], u'mainClass': u'org.apache.spark.examples.SparkPi' } }, # { # u'failureAction': u'INTERRUPT_JOB_QUEUE', # u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51', # u'sparkJob': { # u'applicationArguments': ['10'], # u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'], # u'mainClass': u'org.apache.spark.examples.SparkPi' # } # }, # { # u'failureAction': u'INTERRUPT_JOB_QUEUE', # u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51', # u'sparkJob': { # u'applicationArguments': [u'filesystems3.conf'], # u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'], # u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain', # u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf' # } # } ], u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410', u'publicKey': DEFAULT_PUBLIC_KEY.get(), u'serviceType': u'SPARK', u'workersConfiguration': {}, u'workersGroupSize': u'3' } wf_dir = Submission( request.user, wf, request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy() properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)} properties.update(mapping) submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties) job_id = submission.run() return job_id except RestException as ex: LOG.exception('Error submitting coordinator') raise PopupException(_("Error submitting coordinator %s") % (coordinator, ), detail=ex._headers.get('oozie-error-message', ex), error_code=200)
def get_config(request): config = get_cluster_config(request.user) config['status'] = 0 return JsonResponse(config)