def get_api(request, snippet): from notebook.connectors.dataeng import DataEngApi from notebook.connectors.hiveserver2 import HS2Api from notebook.connectors.jdbc import JdbcApi from notebook.connectors.rdbms import RdbmsApi from notebook.connectors.oozie_batch import OozieApi from notebook.connectors.solr import SolrApi from notebook.connectors.spark_shell import SparkApi from notebook.connectors.spark_batch import SparkBatchApi from notebook.connectors.text import TextApi if snippet.get('wasBatchExecuted'): return OozieApi(user=request.user, request=request) interpreter = [interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type']] if not interpreter: raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet) interpreter = interpreter[0] interface = interpreter['interface'] # Multi cluster cluster = Cluster(request.user) if cluster and cluster.get_type() == 'dataeng': interface = 'dataeng' if interface == 'hiveserver2': return HS2Api(user=request.user, request=request) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': return SparkApi(request.user) elif interface == 'livy-batch': return SparkBatchApi(request.user) elif interface == 'text' or interface == 'markdown': return TextApi(request.user) elif interface == 'rdbms': return RdbmsApi(request.user, interpreter=snippet['type']) elif interface == 'dataeng': return DataEngApi(user=request.user, request=request, cluster_name=cluster.get_interface()) elif interface == 'jdbc': return JdbcApi(request.user, interpreter=interpreter) elif interface == 'solr': return SolrApi(request.user, interpreter=interpreter) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException(_('Notebook connector interface not recognized: %s') % interface)
def apps(self, filters): kwargs = {} if 'time' in filters: if filters['time']['time_unit'] == 'minutes': delta = timedelta(minutes=int(filters['time']['time_value'])) elif filters['time']['time_unit'] == 'hours': delta = timedelta(hours=int(filters['time']['time_value'])) else: delta = timedelta(days=int(filters['time']['time_value'])) kwargs['creation_date_after'] = (datetime.today() - delta).strftime(DATE_FORMAT) # Could also come from filters cluster = Cluster(self.user) if cluster.get_type() == DATAENG: kwargs['cluster_crn'] = cluster.get_id() api = DataEng(self.user) jobs = api.list_jobs(**kwargs) return { 'apps': [{ 'id': app['jobId'], 'name': app['creationDate'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'type': app['jobType'], 'user': '', 'progress': 100, 'duration': 10 * 3600, 'submitted': app['creationDate'] } for app in jobs['jobs']], 'total': len(jobs) }
def _default_interpreters(user): interpreters = [] apps = appmanager.get_apps_dict(user) if 'hive' in apps: interpreters.append(('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'options': {} }),) if 'impala' in apps: interpreters.append(('impala', { 'name': 'Impala', 'interface': 'hiveserver2', 'options': {} }),) if 'pig' in apps: interpreters.append(('pig', { 'name': 'Pig', 'interface': 'oozie', 'options': {} })) if 'oozie' in apps and 'jobsub' in apps: interpreters.extend(( ('java', { 'name': 'Java', 'interface': 'oozie', 'options': {} }), ('spark2', { 'name': 'Spark', 'interface': 'oozie', 'options': {} }), ('mapreduce', { 'name': 'MapReduce', 'interface': 'oozie', 'options': {} }), ('shell', { 'name': 'Shell', 'interface': 'oozie', 'options': {} }), ('sqoop1', { 'name': 'Sqoop 1', 'interface': 'oozie', 'options': {} }), ('distcp', { 'name': 'Distcp', 'interface': 'oozie', 'options': {} }), )) from dashboard.conf import get_properties # Cyclic dependency dashboards = get_properties() if dashboards.get('solr') and dashboards['solr']['analytics']: interpreters.append(('solr', { 'name': 'Solr SQL', 'interface': 'solr', 'options': {} }),) from desktop.models import Cluster # Cyclic dependency cluster = Cluster(user) if cluster and cluster.get_type() == 'dataeng': interpreters.append(('dataeng', { 'name': 'DataEng', 'interface': 'dataeng', 'options': {} })) if 'spark' in apps: interpreters.extend(( ('spark', { 'name': 'Scala', 'interface': 'livy', 'options': {} }), ('pyspark', { 'name': 'PySpark', 'interface': 'livy', 'options': {} }), ('r', { 'name': 'R', 'interface': 'livy', 'options': {} }), ('jar', { 'name': 'Spark Submit Jar', 'interface': 'livy-batch', 'options': {} }), ('py', { 'name': 'Spark Submit Python', 'interface': 'livy-batch', 'options': {} }), ('text', { 'name': 'Text', 'interface': 'text', 'options': {} }), ('markdown', { 'name': 'Markdown', 'interface': 'text', 'options': {} }) )) INTERPRETERS.set_for_testing(OrderedDict(interpreters))