def hue(request):
apps = appmanager.get_apps_dict(request.user)
current_app, other_apps, apps_list = _get_apps(request.user, '')
default_cluster_index, default_cluster_interface = Cluster(request.user).get_list_interface_indexes()
clusters = get_clusters().values()
return render('hue.mako', request, {
'apps': apps,
'other_apps': other_apps,
'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user),
'is_adls_enabled': is_adls_enabled() and has_adls_access(request.user),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(),
'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()),
'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()),
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'banner_message': get_banner_message(request),
'user_preferences': dict((x.key, x.value) for x in UserPreferences.objects.filter(user=request.user)),
'cluster': clusters[0]['type'] if clusters else None,
'clusters_config_json': json.dumps(clusters),
'default_cluster_index': default_cluster_index,
'default_cluster_interface': default_cluster_interface
})
def commonheader(title, section, user, request=None, padding="90px", skip_topbar=False, skip_idle_timeout=False, is_mobile=False):
"""
Returns the rendered common header
"""
current_app, other_apps, apps_list = _get_apps(user, section)
template = 'common_header.mako'
if is_mobile:
template = 'common_header_m.mako'
return django_mako.render_to_string(template, {
'current_app': current_app,
'apps': apps_list,
'other_apps': other_apps,
'title': title,
'section': section,
'padding': padding,
'user': user,
'request': request,
'skip_topbar': skip_topbar,
'skip_idle_timeout': skip_idle_timeout,
'leaflet': {
'layer': desktop.conf.LEAFLET_TILE_LAYER.get(),
'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get(),
'map_options': json.dumps(desktop.conf.LEAFLET_MAP_OPTIONS.get()),
'layer_options': json.dumps(desktop.conf.LEAFLET_TILE_LAYER_OPTIONS.get()),
},
'is_demo': desktop.conf.DEMO_ENABLED.get(),
'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(),
'is_s3_enabled': is_s3_enabled() and has_s3_access(user),
'is_adls_enabled': is_adls_enabled() and has_adls_access(request.user),
'banner_message': get_banner_message(request)
})
def test_export_result(self):
notebook_json = """
{
"selectedSnippet": "hive",
"showHistory": false,
"description": "Test Hive Query",
"name": "Test Hive Query",
"sessions": [
{
"type": "hive",
"properties": [],
"id": null
}
],
"type": "query-hive",
"id": null,
"snippets": [{"id":"2b7d1f46-17a0-30af-efeb-33d4c29b1055","type":"hive","status":"running","statement":"select * from web_logs","properties":{"settings":[],"variables":[],"files":[],"functions":[]},"result":{"id":"b424befa-f4f5-8799-a0b4-79753f2552b1","type":"table","handle":{"log_context":null,"statements_count":1,"end":{"column":21,"row":0},"statement_id":0,"has_more_statements":false,"start":{"column":0,"row":0},"secret":"rVRWw7YPRGqPT7LZ/TeFaA==an","has_result_set":true,"statement":"select * from web_logs","operation_type":0,"modified_row_count":null,"guid":"7xm6+epkRx6dyvYvGNYePA==an"}},"lastExecuted": 1462554843817,"database":"default"}],
"uuid": "d9efdee1-ef25-4d43-b8f9-1a170f69a05a"
}
"""
response = self.client.post(reverse('notebook:export_result'), {
'notebook': notebook_json,
'snippet': json.dumps(json.loads(notebook_json)['snippets'][0]),
'format': json.dumps('hdfs-file'),
'destination': json.dumps('/user/hue'),
'overwrite': json.dumps(False)
})
data = json.loads(response.content)
assert_equal(0, data['status'], data)
assert_equal('/user/hue/Test Hive Query.csv', data['watch_url']['destination'], data)
response = self.client.post(reverse('notebook:export_result'), {
'notebook': notebook_json,
'snippet': json.dumps(json.loads(notebook_json)['snippets'][0]),
'format': json.dumps('hdfs-file'),
'destination': json.dumps('/user/hue/path.csv'),
'overwrite': json.dumps(False)
})
data = json.loads(response.content)
assert_equal(0, data['status'], data)
assert_equal('/user/hue/path.csv', data['watch_url']['destination'], data)
if is_adls_enabled():
response = self.client.post(reverse('notebook:export_result'), {
'notebook': notebook_json,
'snippet': json.dumps(json.loads(notebook_json)['snippets'][0]),
'format': json.dumps('hdfs-file'),
'destination': json.dumps('adl:/user/hue/path.csv'),
'overwrite': json.dumps(False)
})
data = json.loads(response.content)
assert_equal(0, data['status'], data)
assert_equal('adl:/user/hue/path.csv', data['watch_url']['destination'], data)
def is_enabled(fs=None):
if fs == 'hdfs':
return has_hdfs_enabled()
elif fs == 'adl':
return is_adls_enabled()
elif fs == 's3a':
return is_s3_enabled()
elif fs == 'abfs':
return is_abfs_enabled()
import aws
import azure.client
from azure.conf import is_adls_enabled
from aws.conf import is_enabled as is_s3_enabled
from desktop.lib.fs import ProxyFS
from hadoop import cluster
FS_CACHE = {}
DEFAULT_SCHEMA = 'hdfs'
FS_GETTERS = {
"hdfs": cluster.get_hdfs,
"s3a": aws.get_s3fs if is_s3_enabled() else None,
"adl": azure.client.get_client if is_adls_enabled() else None
}
def get_filesystem(name='default'):
"""
Return the filesystem with the given name.
If the filesystem is not defined, raises KeyError
"""
if name not in FS_CACHE:
FS_CACHE[name] = _make_fs(name)
return FS_CACHE[name]
def _make_fs(name):
fs_dict = {}
from desktop.lib.fs import ProxyFS
FS_CACHE = {}
DEFAULT_SCHEMA = None
FS_GETTERS = {}
if has_hdfs_enabled():
FS_GETTERS['hdfs'] = get_hdfs
DEFAULT_SCHEMA = 'hdfs'
if is_s3_enabled():
FS_GETTERS['s3a'] = aws.get_s3fs
if DEFAULT_SCHEMA is None:
DEFAULT_SCHEMA = 's3a'
if is_adls_enabled():
FS_GETTERS['adl'] = azure.client.get_client
if DEFAULT_SCHEMA is None:
DEFAULT_SCHEMA = 'adl'
if is_abfs_enabled():
FS_GETTERS['abfs'] = azure.client.get_client_abfs
if DEFAULT_SCHEMA is None:
DEFAULT_SCHEMA = 'abfs'
def get_filesystem(name='default'):
"""
Return the filesystem with the given name.
If the filesystem is not defined, raises KeyError
"""
if name not in FS_CACHE:
