def create_cluster(request): response = {'status': -1} is_k8 = request.POST.get('is_k8') == 'true' cluster_name = request.POST.get('cluster_name') or 'Analytic Cluster' cdh_version = request.POST.get('cdh_version') public_key = request.POST.get('public_key') instance_type = request.POST.get('instance_type', "workers_group_size" '') environment_name = request.POST.get('environment_name') workers_group_size = int(request.POST.get('workers_group_size', '3')) namespace_name = request.POST.get('namespace_name', 'null') api = DataWarehouse2Api(request.user) if is_k8 else AnalyticDbApi( request.user) data = api.create_cluster(cloud_provider='aws', cluster_name=cluster_name, cdh_version=cdh_version, public_key=public_key, instance_type=instance_type, environment_name=environment_name, namespace_name=namespace_name, workers_group_size=workers_group_size) if data: response['status'] = 0 response['data'] = data else: response['message'] = 'Data Warehouse API: %s' % data['details'] return JsonResponse(response)
def get_context_namespaces(request, interface): ''' Namespaces are node cluster contexts (e.g. Hive + Ranger) that can be queried by computes. ''' response = {} namespaces = [] clusters = list(get_clusters(request.user).values()) # Currently broken if not sent namespaces.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'status': 'CREATED', 'computes': [cluster] } for cluster in clusters if cluster.get('type') == 'direct' ]) if interface == 'hive' or interface == 'impala' or interface == 'report': if get_cluster_config(request.user)['has_computes']: # Note: attaching computes to namespaces might be done via the frontend in the future if interface == 'impala': if IS_K8S_ONLY.get(): adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] for _cluster in adb_clusters: # Add "fake" namespace if needed if not _cluster.get('namespaceCrn'): _cluster['namespaceCrn'] = _cluster['crn'] _cluster['id'] = _cluster['crn'] _cluster['namespaceName'] = _cluster['clusterName'] _cluster['name'] = _cluster['clusterName'] _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8S_ONLY.get() else '', else: adb_clusters = [] if IS_K8S_ONLY.get(): sdx_namespaces = [] else: sdx_namespaces = SdxApi(request.user).list_namespaces() # Adding "fake" namespace for cluster without one sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])]) namespaces.extend([{ 'id': namespace.get('crn', 'None'), 'name': namespace.get('namespaceName'), 'status': namespace.get('status'), 'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')] } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get() ]) response[interface] = namespaces response['status'] = 0 return JsonResponse(response)
def get_context_namespaces(request, interface): response = {} namespaces = [] clusters = get_clusters(request.user).values() namespaces.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'status': 'CREATED', 'computes': [cluster] } for cluster in clusters if cluster.get('type') == 'direct' and cluster['interface'] in (interface, 'all') ]) if interface == 'hive' or interface == 'impala' or interface == 'report': # From Altus SDX if [cluster for cluster in clusters if 'altus' in cluster['type']]: # Note: attaching computes to namespaces might be done via the frontend in the future if interface == 'impala': if IS_K8S_ONLY.get(): adb_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: adb_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] for _cluster in adb_clusters: # Add "fake" namespace if needed if not _cluster.get('namespaceCrn'): _cluster['namespaceCrn'] = _cluster['crn'] _cluster['id'] = _cluster['crn'] _cluster['namespaceName'] = _cluster['clusterName'] _cluster['name'] = _cluster['clusterName'] _cluster['compute_end_point'] = '%(publicHost)s' % _cluster['coordinatorEndpoint'] if IS_K8S_ONLY.get() else '', else: adb_clusters = [] if IS_K8S_ONLY.get(): sdx_namespaces = [] else: sdx_namespaces = SdxApi(request.user).list_namespaces() # Adding "fake" namespace for cluster without one sdx_namespaces.extend([_cluster for _cluster in adb_clusters if not _cluster.get('namespaceCrn') or (IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])]) namespaces.extend([{ 'id': namespace.get('crn', 'None'), 'name': namespace.get('namespaceName'), 'status': namespace.get('status'), 'computes': [_cluster for _cluster in adb_clusters if _cluster.get('namespaceCrn') == namespace.get('crn')] } for namespace in sdx_namespaces if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get() ]) response[interface] = namespaces response['status'] = 0 return JsonResponse(response)
def get_context_computes(request, interface): response = {} computes = [] clusters = get_clusters(request.user).values() has_altus_clusters = [cluster for cluster in clusters if 'altus' in cluster['type']] computes.extend([{ 'id': cluster['id'], 'name': cluster['name'], 'namespace': cluster['id'], 'interface': interface, 'type': cluster['type'] } for cluster in clusters if cluster.get('type') == 'direct' and cluster['interface'] in (interface, 'all') ]) if has_altus_clusters: if interface == 'impala' or interface == 'report': if IS_K8S_ONLY.get(): dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters'] else: dw_clusters = AnalyticDbApi(request.user).list_clusters()['clusters'] computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'namespace': cluster.get('namespaceCrn', cluster.get('crn')), 'compute_end_point': IS_K8S_ONLY.get() and '%(publicHost)s' % cluster['coordinatorEndpoint'] or '', 'type': 'altus-dw' } for cluster in dw_clusters if (cluster.get('status') == 'CREATED' and cluster.get('cdhVersion') >= 'CDH515') or (IS_K8S_ONLY.get() and 'TERMINAT' not in cluster['status'])] ) if interface == 'oozie' or interface == 'spark2': computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'environmentType': cluster.get('environmentType'), 'serviceType': cluster.get('serviceType'), 'namespace': cluster.get('namespaceCrn'), 'type': 'altus-de' } for cluster in DataEngApi(request.user).list_clusters()['clusters']] ) # TODO if interface == 'spark2' keep only SPARK type response[interface] = computes response['status'] = 0 return JsonResponse(response)
def update_cluster(request): response = {'status': -1} cluster_name = request.POST.get('cluster_name') or 'Analytic Cluster' workers_group_size = int(request.POST.get('workers_group_size', '3')) api = DataWarehouse2Api(request.user) data = api.update_cluster(cluster_name=cluster_name, workers_group_size=workers_group_size) if data: response['status'] = 0 response['data'] = data else: response['message'] = 'Data Warehouse API: %s' % data['details'] return JsonResponse(response)
def get_context_computes(request, interface): ''' Some clusters like Snowball can have multiple computes for a certain languages (Hive, Impala...). ''' response = {} computes = [] clusters = get_clusters(request.user).values() has_altus_clusters = [ cluster for cluster in clusters if 'altus' in cluster['type'] or 'snowball' in cluster['type'] ] if has_altus_clusters: if interface == 'impala' or interface == 'report': if IS_K8S_ONLY.get(): dw_clusters = DataWarehouse2Api( request.user).list_clusters()['clusters'] computes.extend([{ 'id': cluster.get('crn'), 'name': cluster.get('clusterName'), 'status': cluster.get('status'), 'namespace': cluster.get('namespaceCrn', cluster.get('crn')), 'compute_end_point': IS_K8S_ONLY.get() and '%(publicHost)s' % cluster['coordinatorEndpoint'] or '', 'type': 'altus-dw' } for cluster in dw_clusters]) response[interface] = computes response['status'] = 0 return JsonResponse(response)
def update_cluster(request): response = {'status': -1} cluster_name = request.POST.get('cluster_name') or 'Analytic Cluster' auto_resize_changed = request.POST.get('auto_resize_changed') == 'true' params = { 'clusterName': cluster_name, 'updateClusterAutoResizeChanged': auto_resize_changed } if auto_resize_changed: updateClusterAutoResize = request.POST.get( 'auto_resize_enabled') == 'true' params['updateClusterAutoResize'] = updateClusterAutoResize if updateClusterAutoResize: params['updateClusterAutoResizeMax'] = int( request.POST.get('auto_resize_max')) if request.POST.get('auto_resize_min'): params['updateClusterAutoResizeMin'] = int( request.POST.get('auto_resize_min')) if request.POST.get('auto_resize_cpu'): params['updateClusterAutoResizeCpu'] = int( request.POST.get('auto_resize_cpu')) else: params['workerReplicas'] = int( request.POST.get('workers_group_size', '3')) api = DataWarehouse2Api(request.user) data = api.update_cluster(**params) if data: response['status'] = 0 response['data'] = data else: response['message'] = 'Data Warehouse API: %s' % data['details'] return JsonResponse(response)
def __init__(self, user, version=1): super(ClusterApi, self).__init__(user) self.version = version self.api = DataWarehouse2Api(self.user)
def __init__(self, user, version=1): super(DataWarehouseClusterApi, self).__init__(user) self.version = version self.api = DataWarehouse2Api( self.user) if version == 2 else AnalyticDbApi(self.user)