def test_get_interpreters_to_show():
default_interpreters = OrderedDict((
('hive', {
'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive'
}),
('spark', {
'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala'
}),
('pig', {
'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig'
}),
('java', {
'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'java'
})
))
expected_interpreters = OrderedDict((
('java', {
'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'java'
}),
('pig', {
'name': 'Pig', 'interface': 'pig', 'is_sql': False, 'type': 'pig', 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig'
}),
('hive', {
'name': 'Hive', 'interface': 'hiveserver2', 'is_sql': True, 'type': 'hive', 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive'
}),
('spark', {
'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala'
})
))
try:
resets = [INTERPRETERS.set_for_testing(default_interpreters), APP_BLACKLIST.set_for_testing('')]
appmanager.DESKTOP_MODULES = []
appmanager.DESKTOP_APPS = None
appmanager.load_apps(APP_BLACKLIST.get())
interpreters_shown_on_wheel_unset = get_ordered_interpreters()
assert_equal(
list(default_interpreters.values()),
interpreters_shown_on_wheel_unset,
'get_interpreters_to_show should return the same as get_interpreters when interpreters_shown_on_wheel is unset. expected: %s, actual: %s' % (
list(default_interpreters.values()), interpreters_shown_on_wheel_unset
)
)
resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig'))
assert_equal(list(expected_interpreters.values()), get_ordered_interpreters(),
'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s'
% (list(expected_interpreters.values()), get_ordered_interpreters()))
finally:
for reset in resets:
reset()
appmanager.DESKTOP_MODULES = []
appmanager.DESKTOP_APPS = None
appmanager.load_apps(APP_BLACKLIST.get())
def _get_interpreter_from_dialect(dialect, user):
if not dialect:
interpreter = get_ordered_interpreters(user=user)[0]
elif '-' in dialect:
interpreter = {
'dialect': dialect.split('-')[0],
'type': dialect.split('-')[1] # Id
}
else:
interpreter = [
i for i in get_ordered_interpreters(user=user)
if i['dialect'] == dialect
][0]
return interpreter
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_id: # Open existing saved editor document
editor_type = _get_editor_type(editor_id)
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(
template, request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'editor_type':
editor_type,
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def get_engines(user):
engines = []
apps = get_apps_dict()
settings = get_properties()
if 'search' in apps:
engines.append({
'name':
_('index (Solr)'),
'type':
'solr',
'analytics':
settings.get('solr') and settings['solr'].get('analytics'),
'nesting':
settings.get('solr') and settings['solr'].get('nesting'),
})
if HAS_SQL_ENABLED.get() and ('beeswax' in apps or 'rdbms' in apps):
engines += [{
'name':
_('table (%s)') % interpreter['name'],
'type':
interpreter['type'],
'async':
interpreter['interface'] == 'hiveserver2',
'analytics':
settings.get('sql') and settings['sql'].get('analytics'),
'nesting':
settings.get('sql') and settings['sql'].get('nesting'),
} for interpreter in get_ordered_interpreters(user)
if interpreter['interface'] in ('hiveserver2', 'jdbc',
'rdbms')]
return engines
def get_interpreter(connector_type, user=None):
interpreter = [
interpreter for interpreter in get_ordered_interpreters(user) if connector_type == interpreter['type']
]
if not interpreter:
if connector_type == 'hbase': # TODO move to connectors
interpreter = [{
'name': 'hbase',
'type': 'hbase',
'interface': 'hbase',
'options': {},
'is_sql': False
}]
elif connector_type == 'kafka':
interpreter = [{
'name': 'kafka',
'type': 'kafka',
'interface': 'kafka',
'options': {},
'is_sql': False
}]
elif connector_type == 'solr':
interpreter = [{
'name': 'solr',
'type': 'solr',
'interface': 'solr',
'options': {},
'is_sql': False
}]
else:
raise PopupException(_('Snippet type %s is not configured.') % connector_type)
elif len(interpreter) > 1:
raise PopupException(_('Snippet type %s matching more than one interpreter: %s') % (connector_type, len(interpreter)))
return interpreter[0]
def hue(request):
apps = appmanager.get_apps_dict(request.user)
return render(
'hue.mako', request, {
'apps':
apps,
'interpreters':
get_ordered_interpreters(request.user),
'is_s3_enabled':
is_s3_enabled() and has_s3_access(request.user),
'is_ldap_setup':
'desktop.auth.backend.LdapBackend'
in desktop.conf.AUTH.BACKEND.get(),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution':
desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo':
desktop.conf.DEMO_ENABLED.get(),
'banner_message':
get_banner_message(request),
'cluster_config':
ClusterConfig(request.user),
'user_preferences':
dict((x.key, x.value)
for x in UserPreferences.objects.filter(user=request.user))
})
def browse(request, database, table, partition_spec=None):
snippet = {'type': request.POST.get('sourceType', 'hive')}
statement = get_api(request,
snippet).get_browse_query(snippet, database, table,
partition_spec)
editor_type = snippet['type']
if request.method == 'POST':
notebook = make_notebook(name='Execute and watch',
editor_type=editor_type,
statement=statement,
status='ready-execute',
is_task=True)
return JsonResponse(notebook.execute(request, batch=False))
else:
editor = make_notebook(name='Browse',
editor_type=editor_type,
statement=statement,
status='ready-execute')
return render(
'editor.mako', request, {
'notebooks_json':
json.dumps([editor.get_data()]),
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'editor_type': editor_type
}),
'editor_type':
editor_type,
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get("editor")
editor_type = request.GET.get("type", "hive")
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit("-", 1)[-1]
template = "editor.mako"
if is_mobile:
template = "editor_m.mako"
if is_embeddable:
template = "editor_embeddable.mako"
return render(
template,
request,
{
"editor_id": editor_id or None,
"notebooks_json": "{}",
"editor_type": editor_type,
"options_json": json.dumps(
{
"languages": get_ordered_interpreters(request.user),
"mode": "editor",
"is_optimizer_enabled": has_optimizer(),
"is_navigator_enabled": has_navigator(request.user),
"editor_type": editor_type,
"mobile": is_mobile,
}
),
},
)
def jdbc_db_list(request):
format_ = {'data': [], 'status': 1}
interpreters = get_ordered_interpreters(request.user)
format_['data'] = [{'value': key['name'], 'name': key['name']} for key in interpreters if key['interface'] == 'jdbc']
format_['status'] = 0
return JsonResponse(format_)
def notebook(request, is_embeddable=False):
notebook_id = request.GET.get("notebook")
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception("Spark is not enabled")
template = "notebook.mako"
if is_embeddable:
template = "notebook_embeddable.mako"
return render(
template,
request,
{
"editor_id": notebook_id or None,
"notebooks_json": "{}",
"options_json": json.dumps(
{
"languages": get_ordered_interpreters(request.user),
"session_properties": SparkApi.get_properties(),
"is_optimizer_enabled": has_optimizer(),
"is_navigator_enabled": has_navigator(request.user),
"editor_type": "notebook",
}
),
"is_yarn_mode": is_yarn_mode,
},
)
def hue(request):
apps = appmanager.get_apps_dict(request.user)
return render(
'hue.mako', request, {
'apps':
apps,
'tours_and_tutorials':
Settings.get_settings().tours_and_tutorials,
'interpreters':
get_ordered_interpreters(request.user),
'is_s3_enabled':
is_s3_enabled() and has_s3_access(request.user),
'is_ldap_setup':
'desktop.auth.backend.LdapBackend'
in desktop.conf.AUTH.BACKEND.get(),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution':
desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get()
},
'is_demo':
desktop.conf.DEMO_ENABLED.get(),
'banner_message':
get_banner_message(request),
'cluster_config':
ClusterConfig(request.user)
})
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_id: # Open existing saved editor document
document = Document2.objects.get(id=editor_id)
editor_type = document.type.rsplit('-', 1)[-1]
template = 'editor.mako'
if is_mobile:
template = 'editor_m.mako'
return render(template, request, {
'editor_id': editor_id or None,
'notebooks_json': '{}',
'is_embeddable': request.GET.get('is_embeddable', False),
'editor_type': editor_type,
'options_json': json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def test_get_ordered_interpreters(self):
with patch('desktop.lib.connectors.api._get_installed_connectors'
) as _get_installed_connectors:
_get_installed_connectors.return_value = [{
'nice_name': 'Hive',
'name': 'hive-1',
'dialect': 'hive',
'category': 'editor',
'interface': 'hiveserver2',
'settings': {},
'dialect_properties': {
'sql_identifier_quote': '`',
'is_sql': True
},
}]
interpreters = get_ordered_interpreters(user=self.user)
assert_true(interpreters, interpreters)
assert_true(
all([
'dialect_properties' in interpreter
for interpreter in interpreters
]), interpreters)
assert_true(
any([
interpreter.get('dialect_properties').get(
'sql_identifier_quote') for interpreter in interpreters
]), interpreters)
def notebook(request, is_embeddable=False):
if not SHOW_NOTEBOOKS.get():
return serve_403_error(request)
notebook_id = request.GET.get('notebook', request.GET.get('editor'))
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
return render('notebook.mako', request, {
'editor_id': notebook_id or None,
'notebooks_json': '{}',
'is_embeddable': request.GET.get('is_embeddable', False),
'options_json': json.dumps({
'languages': get_ordered_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'editor_type': 'notebook'
}),
'is_yarn_mode': is_yarn_mode,
})
def notebook(request, is_embeddable=False):
notebook_id = request.GET.get('notebook')
is_yarn_mode = False
try:
from spark.conf import LIVY_SERVER_SESSION_KIND
is_yarn_mode = LIVY_SERVER_SESSION_KIND.get()
except:
LOG.exception('Spark is not enabled')
template = 'notebook.mako'
if is_embeddable:
template = 'notebook_embeddable.mako'
return render(
template, request, {
'editor_id':
notebook_id or None,
'notebooks_json':
'{}',
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'session_properties': SparkApi.get_properties(),
'is_optimizer_enabled': has_optimizer(),
'is_navigator_enabled': has_navigator(request.user),
'editor_type': 'notebook'
}),
'is_yarn_mode':
is_yarn_mode,
})
def install_custom_examples():
if EXAMPLES.AUTO_LOAD.get():
from desktop.auth.backend import rewrite_user
from beeswax.management.commands import beeswax_install_examples
from useradmin.models import install_sample_user
user = rewrite_user(install_sample_user())
if has_connectors():
interpreters = [{
'type': connector['id'],
'dialect': connector['dialect']
} for connector in _get_installed_connectors(category='editor')]
else:
interpreters = [
{
'type': interpreter['dialect'],
'dialect': interpreter['dialect']
} for interpreter in get_ordered_interpreters(user)
# Only for hive/impala currently, would also need to port to Notebook install examples.
if interpreter['dialect'] in ('hive', 'impala')
]
queries = EXAMPLES.QUERIES.get()
tables = EXAMPLES.TABLES.get(
) # No-op. Only for the saved query samples, not the tables currently.
LOG.info(
'Installing custom examples queries: %(queries)s, tables: %(tables)s for dialects %(dialects)s '
'belonging to user %(user)s' % {
'queries':
queries,
'tables':
tables,
'dialects':
[interpreter['dialect'] for interpreter in interpreters],
'user':
user
})
result = []
for interpreter in interpreters:
successes, errors = beeswax_install_examples.Command().handle(
dialect=interpreter['dialect'],
user=user,
interpreter=interpreter,
queries=queries,
tables=tables,
request=None)
LOG.info(
'Dialect %(dialect)s installed samples: %(successes)s, %(errors)s,'
% {
'dialect': interpreter['dialect'],
'successes': successes,
'errors': errors,
})
result.append((successes, errors))
return result
def test_get_interpreters_to_show():
default_interpreters = OrderedDict((
('hive', {
'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'options': {}
}),
('spark', {
'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'options': {}
}),
('pig', {
'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'options': {}
}),
('java', {
'name': 'Java', 'interface': 'oozie', 'type': 'java', 'options': {}
})
))
expected_interpreters = OrderedDict((
('java', {
'name': 'Java', 'interface': 'oozie', 'type': 'java', 'options': {}
}),
('pig', {
'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'options': {}
}),
('hive', {
'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'options': {}
}),
('spark', {
'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'options': {}
})
))
try:
resets = [INTERPRETERS.set_for_testing(default_interpreters)]
interpreters_shown_on_wheel_unset = get_ordered_interpreters()
assert_equal(default_interpreters.values(), interpreters_shown_on_wheel_unset,
'get_interpreters_to_show should return the same as get_interpreters when '
'interpreters_shown_on_wheel is unset. expected: %s, actual: %s'
% (default_interpreters.values(), interpreters_shown_on_wheel_unset))
resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig'))
assert_equal(expected_interpreters.values(), get_ordered_interpreters(),
'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s'
% (expected_interpreters.values(), get_ordered_interpreters()))
finally:
for reset in resets:
reset()
def test_get_interpreters_to_show():
default_interpreters = OrderedDict((
('hive', {
'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True, 'options': {}
}),
('spark', {
'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}
}),
('pig', {
'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {}
}),
('java', {
'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}
})
))
expected_interpreters = OrderedDict((
('java', {
'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}
}),
('pig', {
'name': 'Pig', 'interface': 'pig', 'is_sql': False, 'type': 'pig', 'options': {}
}),
('hive', {
'name': 'Hive', 'interface': 'hiveserver2', 'is_sql': True, 'type': 'hive', 'options': {}
}),
('spark', {
'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}
})
))
try:
resets = [INTERPRETERS.set_for_testing(default_interpreters)]
interpreters_shown_on_wheel_unset = get_ordered_interpreters()
assert_equal(default_interpreters.values(), interpreters_shown_on_wheel_unset,
'get_interpreters_to_show should return the same as get_interpreters when '
'interpreters_shown_on_wheel is unset. expected: %s, actual: %s'
% (default_interpreters.values(), interpreters_shown_on_wheel_unset))
resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig'))
assert_equal(expected_interpreters.values(), get_ordered_interpreters(),
'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s'
% (expected_interpreters.values(), get_ordered_interpreters()))
finally:
for reset in resets:
reset()
def responsive(request):
apps = appmanager.get_apps_dict(request.user)
return render('responsive.mako', request, {
'apps': apps,
'tours_and_tutorials': Settings.get_settings().tours_and_tutorials,
'interpreters': get_ordered_interpreters(request.user),
'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get()
})
def get_db_component(request):
format_ = {'data': [], 'status': 1, 'message': ''}
db = None
try:
source = json.loads(request.POST.get('source', '{}'))
user = User.objects.get(username=request.user)
name = None
if source['rdbmsMode'] == 'configRdbms':
if source['rdbmsType'] != 'jdbc':
query_server = rdbms.get_query_server_config(server=source['rdbmsType'])
db = rdbms.get(user, query_server=query_server)
else:
interpreters = get_ordered_interpreters(request.user)
options = {}
key = [key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName']]
if key:
options = key[0]['options']
db = Jdbc(driver_name=options['driver'], url=options['url'], username=options['user'], password=options['password'])
else:
name = source['rdbmsType']
if name != 'jdbc':
query_server = {
'server_name': name,
'server_host': source['rdbmsHostname'],
'server_port': int(source['rdbmsPort']),
'username': source['rdbmsUsername'],
'password': source['rdbmsPassword'],
'options': {},
'alias': name
}
db = rdbms.get(user, query_server=query_server)
else:
db = Jdbc(driver_name=source['rdbmsJdbcDriver'], url=source['rdbmsHostname'], username=source['rdbmsUsername'], password=source['rdbmsPassword'])
if source['rdbmsType'] != 'jdbc':
assist = Assist(db)
else:
assist = JdbcAssist(db)
if not source['rdbmsDatabaseName']:
data = assist.get_databases()
elif source['rdbmsDatabaseName']:
data = assist.get_tables(source['rdbmsDatabaseName'])
format_['data'] = [{'name': element, 'value': element} for element in data]
format_['status'] = 0
except Exception, e:
message = _('Error accessing the database %s: %s') % (name, e)
LOG.warn(message)
format['message'] = message
def get_engines(user):
engines = [{'name': _('index (Solr)'), 'type': 'solr'}]
if ENABLE_SQL.get():
engines += [{
'name': _('table (%s)') % interpreter['name'],
'type': interpreter['type'],
'async': interpreter['interface'] == 'hiveserver2'
}
for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms')
]
return engines
def editor(request, is_mobile=False, is_embeddable=False):
editor_id = request.GET.get('editor')
editor_type = request.GET.get('type', 'hive')
gist_id = request.GET.get('gist')
if editor_type == 'notebook' or request.GET.get('notebook'):
return notebook(request)
if editor_type == 'gist':
gist_doc = _get_gist_document(uuid=gist_id)
editor_type = gist_doc.extra
if EXAMPLES.AUTO_OPEN.get() and not editor_id:
sample_query = _get_dialect_example(dialect=editor_type)
if sample_query:
editor_id = sample_query.id
if editor_id and not gist_id: # Open existing saved editor document
editor_type = _get_editor_type(editor_id)
template = 'editor.mako'
if ENABLE_NOTEBOOK_2.get():
template = 'editor2.mako'
elif is_mobile:
template = 'editor_m.mako'
return render(
template, request, {
'editor_id':
editor_id or None,
'notebooks_json':
'{}',
'is_embeddable':
request.GET.get('is_embeddable', False),
'editor_type':
editor_type,
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'is_optimizer_enabled': has_optimizer(),
'is_wa_enabled': has_workload_analytics(),
'is_navigator_enabled': has_catalog(request.user),
'editor_type': editor_type,
'mobile': is_mobile
})
})
def get_api(request, snippet):
from notebook.connectors.dataeng import DataEngApi
from notebook.connectors.hiveserver2 import HS2Api
from notebook.connectors.jdbc import JdbcApi
from notebook.connectors.rdbms import RdbmsApi
from notebook.connectors.oozie_batch import OozieApi
from notebook.connectors.solr import SolrApi
from notebook.connectors.spark_shell import SparkApi
from notebook.connectors.spark_batch import SparkBatchApi
from notebook.connectors.text import TextApi
if snippet.get('wasBatchExecuted'):
return OozieApi(user=request.user, request=request)
interpreter = [interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type']]
if not interpreter:
raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
interpreter = interpreter[0]
interface = interpreter['interface']
# Multi cluster
cluster = Cluster(request.user)
if cluster and cluster.get_type() == 'dataeng':
interface = 'dataeng'
if interface == 'hiveserver2':
return HS2Api(user=request.user, request=request)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
return SparkApi(request.user)
elif interface == 'livy-batch':
return SparkBatchApi(request.user)
elif interface == 'text' or interface == 'markdown':
return TextApi(request.user)
elif interface == 'rdbms':
return RdbmsApi(request.user, interpreter=snippet['type'])
elif interface == 'dataeng':
return DataEngApi(user=request.user, request=request, cluster_name=cluster.get_interface())
elif interface == 'jdbc':
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'solr':
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'pig':
return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4
else:
raise PopupException(_('Notebook connector interface not recognized: %s') % interface)
def _get_db(request):
source = json.loads(
request.POST.get('source', request.POST.get('fileFormat', '{}')))
user = User.objects.get(username=request.user)
name = None
if source['rdbmsMode'] == 'configRdbms':
if source['rdbmsType'] != 'jdbc':
query_server = rdbms.get_query_server_config(
server=source['rdbmsType'])
db = rdbms.get(user, query_server=query_server)
else:
interpreters = get_ordered_interpreters(request.user)
options = {}
key = [
key for key in interpreters
if key['name'] == source['rdbmsJdbcDriverName']
]
if key:
options = key[0]['options']
db = Jdbc(driver_name=options['driver'],
url=options['url'],
username=options['user'],
password=options['password'])
else:
name = source['rdbmsType']
if name != 'jdbc':
query_server = {
'server_name': name,
'server_host': source['rdbmsHostname'],
'server_port': int(source['rdbmsPort']),
'username': source['rdbmsUsername'],
'password': source['rdbmsPassword'],
'options': {},
'alias': name
}
db = rdbms.get(user, query_server=query_server)
else:
db = Jdbc(driver_name=source['rdbmsJdbcDriver'],
url=source['rdbmsHostname'],
username=source['rdbmsUsername'],
password=source['rdbmsPassword'])
return db
def browse(request, database, table):
snippet = {"type": "hive"}
sql_select = get_api(request, snippet).get_select_star_query(snippet, database, table)
editor_type = snippet["type"]
editor = make_notebook(name="Browse", editor_type=editor_type, statement=sql_select, status="ready-execute")
return render(
"editor.mako",
request,
{
"notebooks_json": json.dumps([editor.get_data()]),
"options_json": json.dumps(
{"languages": get_ordered_interpreters(request.user), "mode": "editor", "editor_type": editor_type}
),
"editor_type": editor_type,
},
)
def get_api(request, snippet):
from notebook.connectors.hiveserver2 import HS2Api
from notebook.connectors.jdbc import JdbcApi
from notebook.connectors.rdbms import RdbmsApi
from notebook.connectors.oozie_batch import OozieApi
from notebook.connectors.solr import SolrApi
from notebook.connectors.spark_shell import SparkApi
from notebook.connectors.spark_batch import SparkBatchApi
from notebook.connectors.text import TextApi
if snippet.get('wasBatchExecuted'):
return OozieApi(user=request.user, request=request)
interpreter = [interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type']]
if not interpreter:
raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet)
interpreter = interpreter[0]
interface = interpreter['interface']
if interface == 'hiveserver2':
return HS2Api(user=request.user, request=request)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
return SparkApi(request.user)
elif interface == 'livy-batch':
return SparkBatchApi(request.user)
elif interface == 'text' or interface == 'markdown':
return TextApi(request.user)
elif interface == 'rdbms':
return RdbmsApi(request.user, interpreter=snippet['type'])
elif interface == 'jdbc':
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'solr':
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'pig':
return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4
else:
raise PopupException(_('Notebook connector interface not recognized: %s') % interface)
def browse(request, database, table):
snippet = {'type': 'hive'}
sql_select = get_api(request, snippet).get_select_star_query(
snippet, database, table)
editor_type = snippet['type']
editor = make_notebook(name='Browse',
editor_type=editor_type,
statement=sql_select,
status='ready-execute')
return render(
'editor.mako', request, {
'notebooks_json':
json.dumps([editor.get_data()]),
'options_json':
json.dumps({
'languages': get_ordered_interpreters(request.user),
'mode': 'editor',
'editor_type': editor_type
}),
'editor_type':
editor_type,
})
# limitations under the License.
# Start DBProxy server if we have some JDBC snippets
from notebook.conf import get_ordered_interpreters, ENABLE_DBPROXY_SERVER
def _start_livy_server():
import atexit
import subprocess
import sys
import time
p = subprocess.Popen([sys.executable, sys.argv[0], 'dbproxy_server'])
def cleanup():
p.terminate()
for _ in xrange(5):
if p.poll() == None:
time.sleep(1)
else:
break
else:
p.kill()
atexit.register(cleanup)
if ENABLE_DBPROXY_SERVER.get() and [interpreter for interpreter in get_ordered_interpreters() if interpreter['interface'] == 'jdbc']:
_start_livy_server()
def test_get_ordered_interpreters():
default_interpreters = OrderedDict((
('hive', {
'name': 'Hive', 'displayName': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True,
'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive'
}),
('impala', {
'name': 'Impala', 'displayName': 'Impala', 'interface': 'hiveserver2', 'type': 'impala', 'is_sql': True,
'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'impala'
}),
('spark', {
'name': 'Scala', 'displayName': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {},
'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala'
}),
('pig', {
'name': 'Pig', 'displayName': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {},
'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig'
}),
('java', {
'name': 'Java', 'displayName': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {},
'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'java'
})
))
try:
resets = [APP_BLACKLIST.set_for_testing('')]
appmanager.DESKTOP_MODULES = []
appmanager.DESKTOP_APPS = None
appmanager.load_apps(APP_BLACKLIST.get())
with patch('notebook.conf.is_cm_managed') as is_cm_managed:
with patch('notebook.conf.appmanager.get_apps_dict') as get_apps_dict:
with patch('notebook.conf.has_connectors') as has_connectors:
get_apps_dict.return_value = {'hive': {}}
has_connectors.return_value = False
notebook.conf.INTERPRETERS_CACHE = None
is_cm_managed.return_value = False
# No CM --> Verbatim
INTERPRETERS.set_for_testing(
OrderedDict((
('phoenix', {
'name': 'Phoenix', 'interface': 'sqlalchemy', 'dialect': 'phoenix'
}),)
)
)
assert_equal(
[interpreter['dialect'] for interpreter in get_ordered_interpreters()],
['phoenix']
)
assert_equal( # Check twice because of cache
[interpreter['dialect'] for interpreter in get_ordered_interpreters()],
['phoenix']
)
is_cm_managed.return_value = True
notebook.conf.INTERPRETERS_CACHE = None
# CM --> Append []
INTERPRETERS.set_for_testing(
OrderedDict(()
)
)
assert_equal(
[interpreter['dialect'] for interpreter in get_ordered_interpreters()],
['hive']
)
assert_equal( # Check twice
[interpreter['dialect'] for interpreter in get_ordered_interpreters()],
['hive']
)
notebook.conf.INTERPRETERS_CACHE = None
# CM --> Append [Phoenix]
INTERPRETERS.set_for_testing(
OrderedDict((
('phoenix', {
'name': 'Phoenix', 'interface': 'sqlalchemy', 'dialect': 'phoenix'
}),)
)
)
assert_equal(
[interpreter['dialect'] for interpreter in get_ordered_interpreters()],
['hive', 'phoenix']
)
assert_equal( # Check twice
[interpreter['dialect'] for interpreter in get_ordered_interpreters()],
['hive', 'phoenix']
)
finally:
for reset in resets:
reset()
appmanager.DESKTOP_MODULES = []
appmanager.DESKTOP_APPS = None
appmanager.load_apps(APP_BLACKLIST.get())
notebook.conf.INTERPRETERS_CACHE = None
from notebook.conf import get_ordered_interpreters, ENABLE_DBPROXY_SERVER
def _start_livy_server():
import atexit
import subprocess
import sys
import time
p = subprocess.Popen([sys.executable, sys.argv[0], 'dbproxy_server'])
def cleanup():
p.terminate()
for _ in xrange(5):
if p.poll() == None:
time.sleep(1)
else:
break
else:
p.kill()
atexit.register(cleanup)
if ENABLE_DBPROXY_SERVER.get() and [
interpreter for interpreter in get_ordered_interpreters()
if interpreter['interface'] == 'jdbc'
]:
_start_livy_server()
def create_table_from_file(self, request, source, destination, start_time=-1, dry_run=False):
if '.' in destination['name']:
database, table_name = destination['name'].split('.', 1)
else:
database = 'default'
table_name = destination['name']
final_table_name = table_name
source_type = [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter['dialect'] == 'phoenix'][0]
editor_type = source_type
columns = destination['columns']
# Until we have proper type convertion
for col in columns:
if col['type'] == 'string':
col['type'] = 'varchar'
sql = '''CREATE TABLE IF NOT EXISTS %(table_name)s (
%(columns)s
CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s)
);
''' % {
'database': database,
'table_name': table_name,
'columns': ',\n'.join([' %(name)s %(type)s' % col for col in columns]),
'primary_keys': ', '.join(destination.get('indexerPrimaryKey'))
}
source_path = urllib_unquote(source['path'])
if source['inputFormat'] == 'file':
file_obj = request.fs.open(source_path)
content = file_obj.read().decode("utf-8")
csvfile = string_io(content)
reader = csv.reader(csvfile)
else:
local_file = open(source_path, 'r')
reader = csv.reader(local_file)
if destination['indexerRunJob']:
for count, csv_row in enumerate(reader):
if (source['format']['hasHeader'] and count == 0) or not csv_row:
continue
else:
_sql = ', '.join([ "'{0}'".format(col_val) if columns[count]['type'] in ('varchar', 'timestamp') \
else '{0}'.format(col_val) for count, col_val in enumerate(csv_row)])
sql += '''\nUPSERT INTO %(table_name)s VALUES (%(csv_row)s);\n''' % {
'database': database,
'table_name': table_name,
'csv_row': _sql
}
if dry_run:
return sql
else:
on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \
'?source_type=' + source_type
return make_notebook(
name=_('Creating table %(database)s.%(table)s') % {'database': database, 'table': final_table_name},
editor_type=editor_type,
statement=sql.strip(),
status='ready',
database=database,
on_success_url=on_success_url,
last_executed=start_time,
is_task=True
)
def get_api(request, snippet):
from notebook.connectors.oozie_batch import OozieApi
if snippet.get('wasBatchExecuted'):
return OozieApi(user=request.user, request=request)
if snippet['type'] == 'report':
snippet['type'] = 'impala'
interpreter = [
interpreter for interpreter in get_ordered_interpreters(request.user)
if interpreter['type'] == snippet['type']
]
if not interpreter:
if snippet['type'] == 'hbase':
interpreter = [{
'name': 'hbase',
'type': 'hbase',
'interface': 'hbase',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'kafka':
interpreter = [{
'name': 'kafka',
'type': 'kafka',
'interface': 'kafka',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'solr':
interpreter = [{
'name': 'solr',
'type': 'solr',
'interface': 'solr',
'options': {},
'is_sql': False
}]
else:
raise PopupException(
_('Snippet type %(type)s is not configured in hue.ini') %
snippet)
interpreter = interpreter[0]
interface = interpreter['interface']
# Multi cluster
cluster = json.loads(request.POST.get('cluster', '""')) # Via Catalog API
if cluster == 'undefined':
cluster = None
if not cluster and snippet.get('compute'): # Via notebook.ko.js
cluster = snippet.get('compute').get('id')
if cluster and 'crn:altus:dataware:' in cluster:
interface = 'altus-adb'
if cluster:
LOG.info('Selected cluster %s' % cluster)
if interface == 'hiveserver2':
from notebook.connectors.hiveserver2 import HS2Api
return HS2Api(user=request.user, request=request, cluster=cluster)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
from notebook.connectors.spark_shell import SparkApi
return SparkApi(request.user)
elif interface == 'livy-batch':
from notebook.connectors.spark_batch import SparkBatchApi
return SparkBatchApi(request.user)
elif interface == 'text' or interface == 'markdown':
from notebook.connectors.text import TextApi
return TextApi(request.user)
elif interface == 'rdbms':
from notebook.connectors.rdbms import RdbmsApi
return RdbmsApi(request.user, interpreter=snippet['type'])
elif interface == 'altus-adb':
from notebook.connectors.altus_adb import AltusAdbApi
return AltusAdbApi(user=request.user,
cluster_name=cluster,
request=request)
elif interface == 'dataeng':
from notebook.connectors.dataeng import DataEngApi
return DataEngApi(user=request.user,
request=request,
cluster_name=cluster.get('name'))
elif interface == 'jdbc' or interface == 'teradata':
from notebook.connectors.jdbc import JdbcApi
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'sqlalchemy':
from notebook.connectors.sqlalchemyapi import SqlAlchemyApi
return SqlAlchemyApi(request.user, interpreter=interpreter)
elif interface == 'solr':
from notebook.connectors.solr import SolrApi
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'hbase':
from notebook.connectors.hbase import HBaseApi
return HBaseApi(request.user)
elif interface == 'kafka':
from notebook.connectors.kafka import KafkaApi
return KafkaApi(request.user)
elif interface == 'pig':
return OozieApi(user=request.user,
request=request) # Backward compatibility until Hue 4
else:
raise PopupException(
_('Notebook connector interface not recognized: %s') % interface)
def get_api(request, snippet):
from notebook.connectors.oozie_batch import OozieApi
if snippet.get('wasBatchExecuted'):
return OozieApi(user=request.user, request=request)
if snippet['type'] == 'report':
snippet['type'] = 'impala'
interpreter = [
interpreter for interpreter in get_ordered_interpreters(request.user)
if snippet['type'] in (interpreter['type'], interpreter['interface'])
]
if not interpreter:
if snippet['type'] == 'hbase':
interpreter = [{
'name': 'hbase',
'type': 'hbase',
'interface': 'hbase',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'kafka':
interpreter = [{
'name': 'kafka',
'type': 'kafka',
'interface': 'kafka',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'solr':
interpreter = [{
'name': 'solr',
'type': 'solr',
'interface': 'solr',
'options': {},
'is_sql': False
}]
elif snippet['type'] == 'custom':
interpreter = [{
'name': snippet['name'],
'type': snippet['type'],
'interface': snippet['interface'],
'options': snippet.get('options', {}),
'is_sql': False
}]
else:
raise PopupException(
_('Snippet type %(type)s is not configured.') % snippet)
interpreter = interpreter[0]
interface = interpreter['interface']
if CONNECTORS.IS_ENABLED.get():
cluster = {
'connector': snippet['type'],
'id': interpreter['type'],
}
snippet['type'] = snippet['type'].split('-', 2)[0]
cluster.update(interpreter['options'])
# Multi cluster
elif has_multi_cluster():
cluster = json.loads(request.POST.get(
'cluster',
'""')) # Via Catalog autocomplete API or Notebook create sessions
if cluster == '""' or cluster == 'undefined':
cluster = None
if not cluster and snippet.get('compute'): # Via notebook.ko.js
cluster = snippet['compute']
else:
cluster = None
cluster_name = cluster.get('id') if cluster else None
if cluster and 'altus:dataware:k8s' in cluster_name:
interface = 'hiveserver2'
elif cluster and 'crn:altus:dataware:' in cluster_name:
interface = 'altus-adb'
elif cluster and 'crn:altus:dataeng:' in cluster_name:
interface = 'dataeng'
LOG.info('Selected cluster %s %s interface %s' %
(cluster_name, cluster, interface))
snippet['interface'] = interface
if interface.startswith('hiveserver2') or interface == 'hms':
from notebook.connectors.hiveserver2 import HS2Api
return HS2Api(user=request.user,
request=request,
cluster=cluster,
interface=interface)
elif interface == 'oozie':
return OozieApi(user=request.user, request=request)
elif interface == 'livy':
from notebook.connectors.spark_shell import SparkApi
return SparkApi(request.user)
elif interface == 'livy-batch':
from notebook.connectors.spark_batch import SparkBatchApi
return SparkBatchApi(request.user)
elif interface == 'text' or interface == 'markdown':
from notebook.connectors.text import TextApi
return TextApi(request.user)
elif interface == 'rdbms':
from notebook.connectors.rdbms import RdbmsApi
return RdbmsApi(request.user,
interpreter=snippet['type'],
query_server=snippet.get('query_server'))
elif interface == 'altus-adb':
from notebook.connectors.altus_adb import AltusAdbApi
return AltusAdbApi(user=request.user,
cluster_name=cluster_name,
request=request)
elif interface == 'dataeng':
from notebook.connectors.dataeng import DataEngApi
return DataEngApi(user=request.user,
request=request,
cluster_name=cluster_name)
elif interface == 'jdbc':
if interpreter['options'] and interpreter['options'].get(
'url', '').find('teradata') >= 0:
from notebook.connectors.jdbc_teradata import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
if interpreter['options'] and interpreter['options'].get(
'url', '').find('awsathena') >= 0:
from notebook.connectors.jdbc_athena import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('presto') >= 0:
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interpreter['options'] and interpreter['options'].get(
'url', '').find('clickhouse') >= 0:
from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse
return JdbcApiClickhouse(request.user, interpreter=interpreter)
else:
from notebook.connectors.jdbc import JdbcApi
return JdbcApi(request.user, interpreter=interpreter)
elif interface == 'teradata':
from notebook.connectors.jdbc import JdbcApiTeradata
return JdbcApiTeradata(request.user, interpreter=interpreter)
elif interface == 'athena':
from notebook.connectors.jdbc import JdbcApiAthena
return JdbcApiAthena(request.user, interpreter=interpreter)
elif interface == 'presto':
from notebook.connectors.jdbc_presto import JdbcApiPresto
return JdbcApiPresto(request.user, interpreter=interpreter)
elif interface == 'sqlalchemy':
from notebook.connectors.sqlalchemyapi import SqlAlchemyApi
return SqlAlchemyApi(request.user, interpreter=interpreter)
elif interface == 'solr':
from notebook.connectors.solr import SolrApi
return SolrApi(request.user, interpreter=interpreter)
elif interface == 'hbase':
from notebook.connectors.hbase import HBaseApi
return HBaseApi(request.user)
elif interface == 'kafka':
from notebook.connectors.kafka import KafkaApi
return KafkaApi(request.user)
elif interface == 'pig':
return OozieApi(user=request.user,
request=request) # Backward compatibility until Hue 4
else:
raise PopupException(
_('Notebook connector interface not recognized: %s') % interface)
def run_sqoop(request, source, destination, start_time):
rdbms_mode = source['rdbmsMode']
rdbms_name = source['rdbmsJdbcDriverName'] if source[
'rdbmsType'] == 'jdbc' else source['rdbmsType']
rdbms_database_name = source['rdbmsDatabaseName']
rdbms_all_tables_selected = source['rdbmsAllTablesSelected']
destination_type = destination['outputFormat']
destination_name = destination['name']
destination_table_name = destination['tableName']
destination_database_name = destination['databaseName']
destination_mappers_num = destination['numMappers']
destination_file_output_format = destination['rdbmsFileOutputFormat']
destination_custom_fields_delimiter = destination['customFieldsDelimiter']
destination_custom_line_delimiter = destination['customLineDelimiter']
destination_custom_enclosed_by_delimiter = destination[
'customEnclosedByDelimiter']
destination_splitby_column = destination['rdbmsSplitByColumn']
if not rdbms_all_tables_selected:
rdbms_table_name = source['rdbmsTableName']
if rdbms_mode == 'configRdbms' and rdbms_name == 'jdbc':
username = ''
password = ''
url = ''
interpreters = get_ordered_interpreters(request.user)
key = [
key for key in interpreters
if key['name'] == source['rdbmsJdbcDriverName']
]
if key:
options = key[0]['options']
url = options['url']
password = options['password']
username = options['user']
statement = '--connect %(url)s --username %(username)s --password %(password)s' % {
'url': url,
'username': username,
'password': password
}
else:
if rdbms_mode == 'configRdbms':
rdbms_host = DATABASES[rdbms_name].HOST.get()
rdbms_port = DATABASES[rdbms_name].PORT.get()
rdbms_user_name = DATABASES[rdbms_name].USER.get()
rdbms_password = get_database_password(rdbms_name)
else:
rdbms_host = source['rdbmsHostname']
rdbms_port = source['rdbmsPort']
rdbms_user_name = source['rdbmsUsername']
rdbms_password = source['rdbmsPassword']
password_file_path = request.fs.join(
request.fs.get_home_dir() + '/sqoop/',
uuid.uuid4().hex + '.password')
request.fs.do_as_user(request.user,
request.fs.create,
password_file_path,
overwrite=True,
permission=0700,
data=smart_str(rdbms_password))
lib_files = []
if destination['sqoopJobLibPaths']:
lib_files = [{
'path': f['path'],
'type': 'jar'
} for f in destination['sqoopJobLibPaths']]
statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % {
'rdbmsName': rdbms_name,
'rdbmsHost': rdbms_host,
'rdbmsPort': rdbms_port,
'rdbmsDatabaseName': rdbms_database_name,
'rdbmsUserName': rdbms_user_name,
'passwordFilePath': password_file_path
}
if destination_type == 'file':
success_url = '/filebrowser/view/' + destination_name
targetDir = request.fs.fs_defaultfs + destination_name
if rdbms_all_tables_selected:
statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % {
'statement': statement,
'targetDir': targetDir
}
else:
statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % {
'statement': statement,
'rdbmsTableName': rdbms_table_name,
'targetDir': targetDir
}
if destination_file_output_format == 'text':
statement = '%(statement)s --as-textfile --fields-terminated-by %(customFieldsDelimiter)s --lines-terminated-by %(customLineDelimiter)s --enclosed-by %(customEnclosedByDelimiter)s' % {
'statement':
statement,
'customFieldsDelimiter':
destination_custom_fields_delimiter,
'customLineDelimiter':
destination_custom_line_delimiter,
'customEnclosedByDelimiter':
destination_custom_enclosed_by_delimiter
}
elif destination_file_output_format == 'sequence':
statement = '%(statement)s --as-sequencefile' % {
'statement': statement
}
elif destination_file_output_format == 'avro':
statement = '%(statement)s --as-avrodatafile' % {
'statement': statement
}
statement = _splitby_column_check(statement,
destination_splitby_column)
elif destination_type == 'table':
success_url = reverse('metastore:describe_table',
kwargs={
'database': destination_database_name,
'table': destination_table_name
})
if rdbms_all_tables_selected:
statement = 'import-all-tables %(statement)s --hive-import' % {
'statement': statement
}
else:
statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % {
'statement': statement,
'rdbmsTableName': rdbms_table_name
}
statement = _splitby_column_check(statement,
destination_splitby_column)
elif destination_type == 'hbase':
success_url = '/hbase/#HBase/' + destination_table_name
# Todo
statement = '%(statement)s --num-mappers %(numMappers)s' % {
'statement': statement,
'numMappers': destination_mappers_num
}
task = make_notebook(name=_(
'Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s'
) % {
'rdbmsDatabaseName': rdbms_database_name,
'rdbmsDatabaseName': rdbms_database_name,
'path': destination_name
},
editor_type='sqoop1',
statement=statement,
files=lib_files,
status='ready',
on_success_url=success_url,
last_executed=start_time,
is_task=True)
return task.execute(request, batch=False)