def _submit_coordinator(request, coordinator, mapping):
try:
wf = coordinator.workflow
if IS_MULTICLUSTER_ONLY.get() and has_multi_cluster():
mapping['auto-cluster'] = {
u'additionalClusterResourceTags': [],
u'automaticTerminationCondition':
u'EMPTY_JOB_QUEUE', #'u'NONE',
u'cdhVersion':
u'CDH514',
u'clouderaManagerPassword':
u'guest',
u'clouderaManagerUsername':
u'guest',
u'clusterName':
u'analytics4', # Add time variable
u'computeWorkersConfiguration': {
u'bidUSDPerHr': 0,
u'groupSize': 0,
u'useSpot': False
},
u'environmentName':
u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
u'instanceBootstrapScript':
u'',
u'instanceType':
u'm4.xlarge',
u'jobSubmissionGroupName':
u'',
u'jobs': [
{
u'failureAction': u'INTERRUPT_JOB_QUEUE',
u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
u'sparkJob': {
u'applicationArguments': ['5'],
u'jars': [
u's3a://datawarehouse-customer360/ETL/spark-examples.jar'
],
u'mainClass':
u'org.apache.spark.examples.SparkPi'
}
},
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': ['10'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
# u'mainClass': u'org.apache.spark.examples.SparkPi'
# }
# },
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': [u'filesystems3.conf'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
# u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
# u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'
# }
# }
],
u'namespaceName':
u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
u'publicKey':
DEFAULT_PUBLIC_KEY.get(),
u'serviceType':
u'SPARK',
u'workersConfiguration': {},
u'workersGroupSize':
u'3'
}
wf_dir = Submission(
request.user,
wf,
request.fs,
request.jt,
mapping,
local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user,
coordinator,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") %
(coordinator, ),
detail=ex._headers.get('oozie-error-message', ex),
error_code=200)
def _submit_coordinator(request, coordinator, mapping):
try:
wf = coordinator.workflow
if IS_MULTICLUSTER_ONLY.get() and has_multi_cluster():
mapping['auto-cluster'] = {
u'additionalClusterResourceTags': [],
u'automaticTerminationCondition': u'EMPTY_JOB_QUEUE', #'u'NONE',
u'cdhVersion': u'CDH514',
u'clouderaManagerPassword': u'guest',
u'clouderaManagerUsername': u'guest',
u'clusterName': u'analytics4', # Add time variable
u'computeWorkersConfiguration': {
u'bidUSDPerHr': 0,
u'groupSize': 0,
u'useSpot': False
},
u'environmentName': u'crn:altus:environments:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:environment:analytics/236ebdda-18bd-428a-9d2b-cd6973d42946',
u'instanceBootstrapScript': u'',
u'instanceType': u'm4.xlarge',
u'jobSubmissionGroupName': u'',
u'jobs': [{
u'failureAction': u'INTERRUPT_JOB_QUEUE',
u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
u'sparkJob': {
u'applicationArguments': ['5'],
u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
u'mainClass': u'org.apache.spark.examples.SparkPi'
}
},
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': ['10'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/spark-examples.jar'],
# u'mainClass': u'org.apache.spark.examples.SparkPi'
# }
# },
# {
# u'failureAction': u'INTERRUPT_JOB_QUEUE',
# u'name': u'a87e20d7-5c0d-49ee-ab37-625fa2803d51',
# u'sparkJob': {
# u'applicationArguments': [u'filesystems3.conf'],
# u'jars': [u's3a://datawarehouse-customer360/ETL/envelope-0.6.0-SNAPSHOT-c6.jar'],
# u'mainClass': u'com.cloudera.labs.envelope.EnvelopeMain',
# u'sparkArguments': u'--archives=s3a://datawarehouse-customer360/ETL/filesystems3.conf'
# }
# }
],
u'namespaceName': u'crn:altus:sdx:us-west-1:12a0079b-1591-4ca0-b721-a446bda74e67:namespace:analytics/7ea35fe5-dbc9-4b17-92b1-97a1ab32e410',
u'publicKey': DEFAULT_PUBLIC_KEY.get(),
u'serviceType': u'SPARK',
u'workersConfiguration': {},
u'workersGroupSize': u'3'
}
wf_dir = Submission(request.user, wf, request.fs, request.jt, mapping, local_tz=coordinator.data['properties']['timezone']).deploy()
properties = {'wf_application_path': request.fs.get_hdfs_path(wf_dir)}
properties.update(mapping)
submission = Submission(request.user, coordinator, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting coordinator')
raise PopupException(_("Error submitting coordinator %s") % (coordinator,), detail=ex._headers.get('oozie-error-message', ex), error_code=200)
def get_api(request, snippet):
from notebook.connectors.oozie_batch import OozieApi
if snippet.get('wasBatchExecuted'):
return OozieApi(user=request.user, request=request)
if snippet['type'] == 'report':
snippet['type'] = 'impala'
interpreter = [
interpreter for interpreter in get_ordered_interpreters(request.user)
if snippet['type'] in (interpreter['type'], interpreter['interface'])
]
if not interpreter:
if snippet['type'] == 'hbase':
interpreter = [{
'name': 'hbase',
'type': 'hbase',
'interface': 'hbase',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'kafka':
interpreter = [{
'name': 'kafka',
'type': 'kafka',
'interface': 'kafka',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'solr':
interpreter = [{
'name': 'solr',
'type': 'solr',
'interface': 'solr',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'custom':
interpreter = [{
'name': snippet['name'],
'type': snippet['type'],
'interface': snippet['interface'],
'options': snippet.get('options', {}),
'is_sql': False
}]
else:
raise PopupException(
_('Snippet type %(type)s is not configured.') % snippet)
interpreter = interpreter[0]
interface = interpreter['interface']
if CONNECTORS.IS_ENABLED.get():
cluster = {
'connector': snippet['type'],
'id': interpreter['type'],
}
snippet['type'] = snippet['type'].split('-', 2)[0]
cluster.update(interpreter['options'])
# Multi cluster
elif has_multi_cluster():
cluster = json.loads(request.POST.get(
'cluster',
'""')) # Via Catalog autocomplete API or Notebook create sessions
if cluster == '""' or cluster == 'undefined':
cluster = None
if not cluster and snippet.get('compute'): # Via notebook.ko.js
cluster = snippet['compute']
else:
cluster = None
cluster_name = cluster.get('id') if cluster else None
if cluster and 'altus:dataware:k8s' in cluster_name:
interface = 'hiveserver2'
elif cluster and 'crn:altus:dataware:' in cluster_name:
interface = 'altus-adb'
elif cluster and 'crn:altus:dataeng:' in cluster_name:
interface = 'dataeng'
LOG.info('Selected cluster %s %s interface %s' %
(cluster_name, cluster, interface))
snippet['interface'] = interface
if interface.startswith('hiveserver2') or interface == 'hms':
from notebook.connectors.hiveserver2 import HS2Api
return HS2Api(user=request.user,
request=request,
cluster=cluster,
interface=interface)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
from notebook.connectors.spark_shell import SparkApi
return SparkApi(request.user)
elif interface == 'livy-batch':
from notebook.connectors.spark_batch import SparkBatchApi
return SparkBatchApi(request.user)
elif interface == 'text' or interface == 'markdown':
from notebook.connectors.text import TextApi
return TextApi(request.user)
elif interface == 'rdbms':
from notebook.connectors.rdbms import RdbmsApi
return RdbmsApi(request.user,
interpreter=snippet['type'],
query_server=snippet.get('query_server'))
elif interface == 'altus-adb':
from notebook.connectors.altus_adb import AltusAdbApi
return AltusAdbApi(user=request.user,
cluster_name=cluster_name,
request=request)
elif interface == 'dataeng':
from notebook.connectors.dataeng import DataEngApi
return DataEngApi(user=request.user,
request=request,
cluster_name=cluster_name)
elif interface == 'jdbc':
if interpreter['options'] and interpreter['options'].get(
'url', '').find('teradata') >= 0:
from notebook.connectors.jdbc_teradata import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
if interpreter['options'] and interpreter['options'].get(
'url', '').find('awsathena') >= 0:
from notebook.connectors.jdbc_athena import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('presto') >= 0:
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('clickhouse') >= 0:
from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse
return JdbcApiClickhouse(request.user, interpreter=interpreter)
else:
from notebook.connectors.jdbc import JdbcApi
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'teradata':
from notebook.connectors.jdbc import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
elif interface == 'athena':
from notebook.connectors.jdbc import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interface == 'presto':
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interface == 'sqlalchemy':
from notebook.connectors.sqlalchemyapi import SqlAlchemyApi
return SqlAlchemyApi(request.user, interpreter=interpreter)
elif interface == 'solr':
from notebook.connectors.solr import SolrApi
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'hbase':
from notebook.connectors.hbase import HBaseApi
return HBaseApi(request.user)
elif interface == 'kafka':
from notebook.connectors.kafka import KafkaApi
return KafkaApi(request.user)
elif interface == 'pig':
return OozieApi(user=request.user,
request=request) # Backward compatibility until Hue 4
else:
raise PopupException(
_('Notebook connector interface not recognized: %s') % interface)