def test_location_to_url():
assert_equal('/filebrowser/view/var/lib/hadoop-hdfs',
location_to_url('/var/lib/hadoop-hdfs', False))
assert_equal('/filebrowser/view/var/lib/hadoop-hdfs',
location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs'))
assert_equal('/filebrowser/view/',
location_to_url('hdfs://localhost:8020'))
assert_equal(None, location_to_url('thrift://10.0.0.1:9083'))
def alter_database(request, database):
response = {'status': -1, 'data': ''}
source_type = request.POST.get('source_type', 'hive')
db = _get_db(user=request.user, source_type=source_type)
try:
properties = request.POST.get('properties')
if not properties:
raise PopupException(
_("Alter database requires a properties value of key-value pairs."
))
properties = json.loads(properties)
db.alter_database(database, properties=properties)
db_metadata = db.get_database(database)
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['status'] = 0
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Failed to alter database `%s`: %s") % (database,
ex)
def describe_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
sample_results = None
is_view = table_obj.tableType == 'VIRTUAL_VIEW'
# Don't show samples if it's a view (HUE-526).
if not is_view:
# Show the first few rows
hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj))
query_msg = make_beeswax_query(request, hql)
try:
sample_results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
except:
# Gracefully degrade if we're unable to load the results.
logging.exception("Failed to read table '%s'" % table)
sample_results = None
hdfs_link = location_to_url(request, table_obj.sd.location)
load_form = beeswax.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
top_rows=sample_results and list(parse_results(sample_results.data)) or None,
hdfs_link=hdfs_link,
load_form=load_form,
is_view=is_view
))
def browse_partition(request, database, table, partition_id):
db = dbms.get(request.user)
try:
partition_table = db.describe_partition(database, table, int(partition_id))
uri_path = location_to_url(partition_table.path_location)
return redirect(uri_path)
except Exception, e:
raise PopupException(_('Cannot browse partition'), detail=e.message)
def _replace_hdfs_link(self, match):
try:
return '<a href="%s" target="_blank">%s</a>' % (location_to_url(
match.group(0), strict=False), match.group(0))
except:
LOG.exception('failed to replace hdfs links: %s' %
(match.groups(), ))
return match.group(0)
def browse_partition(request, database, table, partition_id):
db = dbms.get(request.user)
try:
partition_table = db.describe_partition(database, table, int(partition_id))
uri_path = location_to_url(partition_table.path_location)
return redirect(uri_path)
except Exception, e:
raise PopupException(_('Cannot browse partition'), detail=e.message)
def _replace_hdfs_link(self, is_embeddable=False, match=None):
try:
return '<a href="%s">%s</a>' % (location_to_url(
match.group(0), strict=False,
is_embeddable=is_embeddable), match.group(0))
except:
LOG.exception('failed to replace hdfs links: %s' %
(match.groups(), ))
return match.group(0)
def browse_partition(request, database, table, partition_spec):
db = dbms.get(request.user)
try:
decoded_spec = urllib.unquote(partition_spec)
partition_table = db.describe_partition(database, table, decoded_spec)
uri_path = location_to_url(partition_table.path_location)
return redirect(uri_path)
except Exception, e:
raise PopupException(_('Cannot browse partition'), detail=e.message)
def browse_partition(request, database, table, partition_spec):
db = dbms.get(request.user)
try:
decoded_spec = urllib.unquote(partition_spec)
partition_table = db.describe_partition(database, table, decoded_spec)
uri_path = location_to_url(partition_table.path_location)
return redirect(uri_path)
except Exception, e:
raise PopupException(_('Cannot browse partition'), detail=e.message)
def _replace_hdfs_link(self, match):
try:
return '<a href="%s" target="_blank">%s</a>' % (
location_to_url(match.group(0), strict=False),
match.group(0),
)
except:
LOG.exception("failed to replace hdfs links: %s" % (match.groups(),))
return match.group(0)
def browse_partition(request, database, table):
if request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
location = HCatClient(request.user.username).get_partition_location(table, partition_name, db=database)
url = location_to_url(request, location)
result = {'url': url}
return HttpResponse(json.dumps(result))
except Exception as ex:
raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def get_database_metadata(request, database):
db = dbms.get(request.user)
response = {"status": -1, "data": ""}
try:
db_metadata = db.get_database(database)
response["status"] = 0
db_metadata["hdfs_link"] = location_to_url(db_metadata["location"])
response["data"] = db_metadata
except Exception, ex:
response["status"] = 1
response["data"] = _("Cannot get metadata for database: %s") % (database,)
def get_database_metadata(request, database):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
db_metadata = db.get_database(database)
response['status'] = 0
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Cannot get metadata for database: %s") % (database,)
def get_database_metadata(request, database):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
db_metadata = db.get_database(database)
response['status'] = 0
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Cannot get metadata for database %s: %s") % (database, ex)
def browse_partition(request, table):
if request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
location = hcat_client().get_partition_location(table, partition_name)
#location = hcat_client().describe_partition(table, partition_name)
url = location_to_url(request, location)
result = {'url': url}
return HttpResponse(json.dumps(result))
except Exception, ex:
raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def browse_partition(request, database, table, partition_spec):
db = dbms.get(request.user)
try:
decoded_spec = urllib.unquote(partition_spec)
partition_table = db.describe_partition(database, table, decoded_spec)
uri_path = location_to_url(partition_table.path_location)
if request.REQUEST.get("format", "html") == "json":
return JsonResponse({'uri_path': uri_path})
else:
return redirect(uri_path)
except Exception, e:
raise PopupException(_('Cannot browse partition'), detail=e.message)
def browse_partition(request, database, table, partition_spec):
db = dbms.get(request.user)
try:
decoded_spec = urllib.unquote(partition_spec)
partition_table = db.describe_partition(database, table, decoded_spec)
uri_path = location_to_url(partition_table.path_location)
if request.REQUEST.get("format", "html") == "json":
return JsonResponse({'uri_path': uri_path})
else:
return redirect(uri_path)
except Exception, e:
raise PopupException(_('Cannot browse partition'), detail=e.message)
def get_database_metadata(request, database):
response = {'status': -1, 'data': ''}
source_type = request.POST.get('source_type', 'hive')
db = _get_db(user=request.user, source_type=source_type)
try:
db_metadata = db.get_database(database)
response['status'] = 0
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Cannot get metadata for database %s: %s") % (database, ex)
def browse_partition(request, table):
if request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
location = hcat_client().get_partition_location(
table, partition_name)
#location = hcat_client().describe_partition(table, partition_name)
url = location_to_url(request, location)
result = {'url': url}
return HttpResponse(json.dumps(result))
except Exception, ex:
raise PopupException('Browse partition',
title="Browse partition",
detail=str(ex))
def browse_partition(request, database, table):
if request.method == 'POST':
try:
partition_name = request.POST.get('partition_name')
location = HCatClient(
request.user.username).get_partition_location(table,
partition_name,
db=database)
url = location_to_url(request, location)
result = {'url': url}
return HttpResponse(json.dumps(result))
except Exception as ex:
raise PopupException('Browse partition',
title="Browse partition",
detail=str(ex))
def alter_database(request, database):
db = dbms.get(request.user)
response = {'status': -1, 'data': ''}
try:
properties = request.POST.get('properties')
if not properties:
raise PopupException(_("Alter database requires a properties value of key-value pairs."))
properties = json.loads(properties)
db.alter_database(database, properties=properties)
db_metadata = db.get_database(database)
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['status'] = 0
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Failed to alter database `%s`: %s") % (database, ex)
def describe_table(request, table):
table_obj = db_utils.meta_client().get_table("default", table)
# Show the first few rows
hql = "SELECT * FROM `%s`" % (table,)
query_msg = make_beeswax_query(request, hql)
try:
results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0)
except:
# Gracefully degrade if we're unable to load the results.
logging.exception("Failed to read table '%s'" % table)
results = None
hdfs_link = location_to_url(request, table_obj.sd.location)
load_form = beeswax.forms.LoadDataForm(table_obj)
return render("describe_table.mako", request, dict(
table=table_obj,
table_name=table,
top_rows=results and list(parse_results(results.data)) or None,
hdfs_link=hdfs_link,
load_form=load_form
))
def hdfs_link(self):
return location_to_url(self.path_location)
def test_location_to_url():
prefix = '/filebrowser/view='
assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False))
assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs'))
assert_equal(prefix + '/', location_to_url('hdfs://localhost:8020'))
assert_equal(prefix + 's3a%3A//bucket/key', location_to_url('s3a://bucket/key'))
def test_location_to_url():
prefix = '/filebrowser/view='
assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False))
assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs'))
assert_equal(prefix + '/', location_to_url('hdfs://localhost:8020'))
assert_equal(prefix + 's3a%3A//bucket/key', location_to_url('s3a://bucket/key'))
def hdfs_link(self): return location_to_url(self.path_location)
def _make_hdfs_link(self, match):
try:
return '<a href="%s" target="_blank">%s</a>' % (location_to_url(
match.group(0), strict=False), match.group(0))
except:
return match.group(0)
def save_results(request, id):
"""
Save the results of a query to an HDFS directory
"""
id = int(id)
query_history = models.QueryHistory.objects.get(id=id)
if query_history.owner != request.user:
raise PopupException('This action is only available to the user who submitted the query.')
_, state = _get_server_id_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method == 'POST':
# Make sure the result is available.
# Note that we may still hit errors during the actual save
if state != models.QueryHistory.STATE.available:
if state in (models.QueryHistory.STATE.failed, models.QueryHistory.STATE.expired):
msg = 'This query has %s. Results unavailable.' % (state,)
else:
msg = 'The result of this query is not available yet.'
raise PopupException(msg)
form = beeswax.forms.SaveResultsForm(request.POST)
# Cancel goes back to results
if request.POST.get('cancel'):
return format_preserving_redirect(request, '/beeswax/watch/%s' % (id,))
if form.is_valid():
# Do save
# 1. Get the results metadata
assert request.POST.get('save')
handle = QueryHandle(id=query_history.server_id, log_context=query_history.log_context)
try:
result_meta = db_utils.db_client().get_results_metadata(handle)
except QueryNotFoundException, ex:
LOG.exception(ex)
raise PopupException('Cannot find query.')
if result_meta.table_dir:
result_meta.table_dir = request.fs.urlsplit(result_meta.table_dir)[2]
# 2. Check for partitioned tables
if result_meta.table_dir is None:
raise PopupException(
'Saving results from a partitioned table is not supported. '
'You may copy from the HDFS location manually.')
# 3. Actual saving of results
try:
if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR:
# To dir
if result_meta.in_tablename:
raise PopupException(
'Saving results from a table to a directory is not supported. '
'You may copy from the HDFS location manually.')
target_dir = form.cleaned_data['target_dir']
request.fs.rename_star(result_meta.table_dir, target_dir)
LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, target_dir))
query_history.save_state(models.QueryHistory.STATE.expired)
fb_url = location_to_url(request, target_dir, strict=False)
popup = PopupWithJframe('Query results stored in %s' % (target_dir,),
launch_app_name='FileBrowser',
launch_app_url=fb_url)
return render_injected(list_query_history(request), popup)
elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL:
# To new table
try:
return _save_results_ctas(request,
query_history,
form.cleaned_data['target_table'],
result_meta)
except BeeswaxException, bex:
LOG.exception(bex)
error_msg, log = expand_exception(bex)
except IOError, ex:
LOG.exception(ex)
error_msg = str(ex)
def test_location_to_url():
assert_equal('/filebrowser/view/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False))
assert_equal('/filebrowser/view/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs'))
assert_equal('/filebrowser/view/', location_to_url('hdfs://localhost:8020'))
assert_equal(None, location_to_url('thrift://10.0.0.1:9083'))
def _make_hdfs_link(self, match):
try:
return '<a href="%s" target="_blank">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0))
except:
return match.group(0)
