def test_location_to_url(): assert_equal('/filebrowser/view/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False)) assert_equal('/filebrowser/view/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs')) assert_equal('/filebrowser/view/', location_to_url('hdfs://localhost:8020')) assert_equal(None, location_to_url('thrift://10.0.0.1:9083'))
def alter_database(request, database): response = {'status': -1, 'data': ''} source_type = request.POST.get('source_type', 'hive') db = _get_db(user=request.user, source_type=source_type) try: properties = request.POST.get('properties') if not properties: raise PopupException( _("Alter database requires a properties value of key-value pairs." )) properties = json.loads(properties) db.alter_database(database, properties=properties) db_metadata = db.get_database(database) db_metadata['hdfs_link'] = location_to_url(db_metadata['location']) response['status'] = 0 response['data'] = db_metadata except Exception, ex: response['status'] = 1 response['data'] = _("Failed to alter database `%s`: %s") % (database, ex)
def describe_table(request, table): table_obj = db_utils.meta_client().get_table("default", table) sample_results = None is_view = table_obj.tableType == 'VIRTUAL_VIEW' # Don't show samples if it's a view (HUE-526). if not is_view: # Show the first few rows hql = "SELECT * FROM `%s` %s" % (table, _get_browse_limit_clause(table_obj)) query_msg = make_beeswax_query(request, hql) try: sample_results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0) except: # Gracefully degrade if we're unable to load the results. logging.exception("Failed to read table '%s'" % table) sample_results = None hdfs_link = location_to_url(request, table_obj.sd.location) load_form = beeswax.forms.LoadDataForm(table_obj) return render("describe_table.mako", request, dict( table=table_obj, table_name=table, top_rows=sample_results and list(parse_results(sample_results.data)) or None, hdfs_link=hdfs_link, load_form=load_form, is_view=is_view ))
def browse_partition(request, database, table, partition_id): db = dbms.get(request.user) try: partition_table = db.describe_partition(database, table, int(partition_id)) uri_path = location_to_url(partition_table.path_location) return redirect(uri_path) except Exception, e: raise PopupException(_('Cannot browse partition'), detail=e.message)
def _replace_hdfs_link(self, match): try: return '<a href="%s" target="_blank">%s</a>' % (location_to_url( match.group(0), strict=False), match.group(0)) except: LOG.exception('failed to replace hdfs links: %s' % (match.groups(), )) return match.group(0)
def _replace_hdfs_link(self, is_embeddable=False, match=None): try: return '<a href="%s">%s</a>' % (location_to_url( match.group(0), strict=False, is_embeddable=is_embeddable), match.group(0)) except: LOG.exception('failed to replace hdfs links: %s' % (match.groups(), )) return match.group(0)
def browse_partition(request, database, table, partition_spec): db = dbms.get(request.user) try: decoded_spec = urllib.unquote(partition_spec) partition_table = db.describe_partition(database, table, decoded_spec) uri_path = location_to_url(partition_table.path_location) return redirect(uri_path) except Exception, e: raise PopupException(_('Cannot browse partition'), detail=e.message)
def _replace_hdfs_link(self, match): try: return '<a href="%s" target="_blank">%s</a>' % ( location_to_url(match.group(0), strict=False), match.group(0), ) except: LOG.exception("failed to replace hdfs links: %s" % (match.groups(),)) return match.group(0)
def browse_partition(request, database, table): if request.method == 'POST': try: partition_name = request.POST.get('partition_name') location = HCatClient(request.user.username).get_partition_location(table, partition_name, db=database) url = location_to_url(request, location) result = {'url': url} return HttpResponse(json.dumps(result)) except Exception as ex: raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def get_database_metadata(request, database): db = dbms.get(request.user) response = {"status": -1, "data": ""} try: db_metadata = db.get_database(database) response["status"] = 0 db_metadata["hdfs_link"] = location_to_url(db_metadata["location"]) response["data"] = db_metadata except Exception, ex: response["status"] = 1 response["data"] = _("Cannot get metadata for database: %s") % (database,)
def get_database_metadata(request, database): db = dbms.get(request.user) response = {'status': -1, 'data': ''} try: db_metadata = db.get_database(database) response['status'] = 0 db_metadata['hdfs_link'] = location_to_url(db_metadata['location']) response['data'] = db_metadata except Exception, ex: response['status'] = 1 response['data'] = _("Cannot get metadata for database: %s") % (database,)
def get_database_metadata(request, database): db = dbms.get(request.user) response = {'status': -1, 'data': ''} try: db_metadata = db.get_database(database) response['status'] = 0 db_metadata['hdfs_link'] = location_to_url(db_metadata['location']) response['data'] = db_metadata except Exception, ex: response['status'] = 1 response['data'] = _("Cannot get metadata for database %s: %s") % (database, ex)
def browse_partition(request, table): if request.method == 'POST': try: partition_name = request.POST.get('partition_name') location = hcat_client().get_partition_location(table, partition_name) #location = hcat_client().describe_partition(table, partition_name) url = location_to_url(request, location) result = {'url': url} return HttpResponse(json.dumps(result)) except Exception, ex: raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def browse_partition(request, database, table, partition_spec): db = dbms.get(request.user) try: decoded_spec = urllib.unquote(partition_spec) partition_table = db.describe_partition(database, table, decoded_spec) uri_path = location_to_url(partition_table.path_location) if request.REQUEST.get("format", "html") == "json": return JsonResponse({'uri_path': uri_path}) else: return redirect(uri_path) except Exception, e: raise PopupException(_('Cannot browse partition'), detail=e.message)
def get_database_metadata(request, database): response = {'status': -1, 'data': ''} source_type = request.POST.get('source_type', 'hive') db = _get_db(user=request.user, source_type=source_type) try: db_metadata = db.get_database(database) response['status'] = 0 db_metadata['hdfs_link'] = location_to_url(db_metadata['location']) response['data'] = db_metadata except Exception, ex: response['status'] = 1 response['data'] = _("Cannot get metadata for database %s: %s") % (database, ex)
def browse_partition(request, table): if request.method == 'POST': try: partition_name = request.POST.get('partition_name') location = hcat_client().get_partition_location( table, partition_name) #location = hcat_client().describe_partition(table, partition_name) url = location_to_url(request, location) result = {'url': url} return HttpResponse(json.dumps(result)) except Exception, ex: raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def browse_partition(request, database, table): if request.method == 'POST': try: partition_name = request.POST.get('partition_name') location = HCatClient( request.user.username).get_partition_location(table, partition_name, db=database) url = location_to_url(request, location) result = {'url': url} return HttpResponse(json.dumps(result)) except Exception as ex: raise PopupException('Browse partition', title="Browse partition", detail=str(ex))
def alter_database(request, database): db = dbms.get(request.user) response = {'status': -1, 'data': ''} try: properties = request.POST.get('properties') if not properties: raise PopupException(_("Alter database requires a properties value of key-value pairs.")) properties = json.loads(properties) db.alter_database(database, properties=properties) db_metadata = db.get_database(database) db_metadata['hdfs_link'] = location_to_url(db_metadata['location']) response['status'] = 0 response['data'] = db_metadata except Exception, ex: response['status'] = 1 response['data'] = _("Failed to alter database `%s`: %s") % (database, ex)
def describe_table(request, table): table_obj = db_utils.meta_client().get_table("default", table) # Show the first few rows hql = "SELECT * FROM `%s`" % (table,) query_msg = make_beeswax_query(request, hql) try: results = db_utils.execute_and_wait(request.user, query_msg, timeout_sec=5.0) except: # Gracefully degrade if we're unable to load the results. logging.exception("Failed to read table '%s'" % table) results = None hdfs_link = location_to_url(request, table_obj.sd.location) load_form = beeswax.forms.LoadDataForm(table_obj) return render("describe_table.mako", request, dict( table=table_obj, table_name=table, top_rows=results and list(parse_results(results.data)) or None, hdfs_link=hdfs_link, load_form=load_form ))
def hdfs_link(self): return location_to_url(self.path_location)
def test_location_to_url(): prefix = '/filebrowser/view=' assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('/var/lib/hadoop-hdfs', False)) assert_equal(prefix + '/var/lib/hadoop-hdfs', location_to_url('hdfs://localhost:8020/var/lib/hadoop-hdfs')) assert_equal(prefix + '/', location_to_url('hdfs://localhost:8020')) assert_equal(prefix + 's3a%3A//bucket/key', location_to_url('s3a://bucket/key'))
def _make_hdfs_link(self, match): try: return '<a href="%s" target="_blank">%s</a>' % (location_to_url( match.group(0), strict=False), match.group(0)) except: return match.group(0)
def save_results(request, id): """ Save the results of a query to an HDFS directory """ id = int(id) query_history = models.QueryHistory.objects.get(id=id) if query_history.owner != request.user: raise PopupException('This action is only available to the user who submitted the query.') _, state = _get_server_id_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method == 'POST': # Make sure the result is available. # Note that we may still hit errors during the actual save if state != models.QueryHistory.STATE.available: if state in (models.QueryHistory.STATE.failed, models.QueryHistory.STATE.expired): msg = 'This query has %s. Results unavailable.' % (state,) else: msg = 'The result of this query is not available yet.' raise PopupException(msg) form = beeswax.forms.SaveResultsForm(request.POST) # Cancel goes back to results if request.POST.get('cancel'): return format_preserving_redirect(request, '/beeswax/watch/%s' % (id,)) if form.is_valid(): # Do save # 1. Get the results metadata assert request.POST.get('save') handle = QueryHandle(id=query_history.server_id, log_context=query_history.log_context) try: result_meta = db_utils.db_client().get_results_metadata(handle) except QueryNotFoundException, ex: LOG.exception(ex) raise PopupException('Cannot find query.') if result_meta.table_dir: result_meta.table_dir = request.fs.urlsplit(result_meta.table_dir)[2] # 2. Check for partitioned tables if result_meta.table_dir is None: raise PopupException( 'Saving results from a partitioned table is not supported. ' 'You may copy from the HDFS location manually.') # 3. Actual saving of results try: if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR: # To dir if result_meta.in_tablename: raise PopupException( 'Saving results from a table to a directory is not supported. ' 'You may copy from the HDFS location manually.') target_dir = form.cleaned_data['target_dir'] request.fs.rename_star(result_meta.table_dir, target_dir) LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, target_dir)) query_history.save_state(models.QueryHistory.STATE.expired) fb_url = location_to_url(request, target_dir, strict=False) popup = PopupWithJframe('Query results stored in %s' % (target_dir,), launch_app_name='FileBrowser', launch_app_url=fb_url) return render_injected(list_query_history(request), popup) elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL: # To new table try: return _save_results_ctas(request, query_history, form.cleaned_data['target_table'], result_meta) except BeeswaxException, bex: LOG.exception(bex) error_msg, log = expand_exception(bex) except IOError, ex: LOG.exception(ex) error_msg = str(ex)
def _make_hdfs_link(self, match): try: return '<a href="%s" target="_blank">%s</a>' % (location_to_url(match.group(0), strict=False), match.group(0)) except: return match.group(0)