def drop_partition(request, database, table): source_type = request.POST.get('source_type', 'hive') cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, source_type=source_type, cluster=cluster) if request.method == 'POST': partition_specs = request.POST.getlist('partition_selection') partition_specs = [spec for spec in partition_specs] try: if request.GET.get("format", "html") == "json": last_executed = json.loads(request.POST.get('start_time'), '-1') sql = db.drop_partitions(database, table, partition_specs, design=None, generate_ddl_only=True) job = make_notebook( name=_('Drop partition %s') % ', '.join(partition_specs)[:100], editor_type=source_type, statement=sql.strip(), status='ready', database=None, on_success_url='assist.db.refresh', is_task=True, last_executed=last_executed ) return JsonResponse(job.execute(request)) else: design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_partitions(database, table, partition_specs, design) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \ reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table}) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message} raise PopupException(error, title=_("DB Error"), detail=log)
def drop_table(request, database): db = dbms.get(request.user) if request.method == 'POST': try: tables = request.POST.getlist('table_selection') tables_objects = [db.get_table(database, table) for table in tables] skip_trash = request.POST.get('skip_trash') == 'on' if request.POST.get('is_embeddable'): sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True) job = make_notebook( name='Execute and watch', editor_type='hive', statement=sql.strip(), status='ready', database=database, on_success_url='assist.db.refresh', is_task=True ) return JsonResponse(job.execute(request)) else: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database}) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message} raise PopupException(error, title=_("Hive Error"), detail=log)
def execute_query(request, design_id=None): """ View function for executing an arbitrary query. It understands the optional GET/POST params: on_success_url If given, it will be displayed when the query is successfully finished. Otherwise, it will display the view query results page by default. """ authorized_get_design(request, design_id) error_message = None form = QueryForm() action = request.path log = None app_name = get_app_name(request) query_type = SavedQuery.TYPES_MAPPING[app_name] design = safe_get_design(request, query_type, design_id) on_success_url = request.REQUEST.get('on_success_url') databases = [] query_server = get_query_server_config(app_name) db = dbms.get(request.user, query_server) try: databases = get_db_choices(request) except Exception, ex: error_message, log = expand_exception(ex, db)
def drop_database(request): db = dbms.get(request.user) if request.method == 'POST': databases = request.POST.getlist('database_selection') try: design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) if request.POST.get('is_embeddable'): sql = db.drop_databases(databases, design, generate_ddl_only=True) job = make_notebook( name='Execute and watch', editor_type='hive', statement=sql.strip(), status='ready', database=None, on_success_url='assist.db.refresh', is_task=True ) return JsonResponse(job.execute(request)) else: query_history = db.drop_databases(databases, design) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases') return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message} raise PopupException(error, title=_("Hive Error"), detail=log)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the execute_query view. If the result set has has_result_set=False, display an empty result. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See execute_query().) """ first_row = long(first_row) start_over = (first_row == 0) results = type('Result', (object,), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) data = [] fetch_error = False error_message = '' log = '' columns = [] app_name = get_app_name(request) query_history = authorized_get_query_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() db = dbms.get(request.user, query_server) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = parse_query_context(context_param) # Update the status as expired should not be accessible expired = state == models.QueryHistory.STATE.expired # Retrieve query results or use empty result if no result set try: if query_server['server_name'] == 'impala' and not handle.has_result_set: downloadable = False else: results = db.fetch(handle, start_over, 100) # Materialize and HTML escape results data = escape_rows(results.rows()) # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data log = db.get_log(handle) columns = results.data_table.cols() except Exception, ex: LOG.exception('error fetching results') fetch_error = True error_message, log = expand_exception(ex, db, handle)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the watch_query view. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See watch_query().) """ first_row = long(first_row) start_over = (first_row == 0) results = None data = None fetch_error = False error_message = '' log = '' app_name = get_app_name(request) query_history = authorized_get_history(request, id, must_exist=True) db = dbms.get(request.user, query_history.get_query_server_config()) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = _parse_query_context(context_param) # To remove in Hue 2.3 download = request.GET.get('download', '') # Update the status as expired should not be accessible # Impala does not support startover for now expired = state == models.QueryHistory.STATE.expired if expired or app_name == 'impala': state = models.QueryHistory.STATE.expired query_history.save_state(state) # Retrieve query results try: if not download: results = db.fetch(handle, start_over, 100) data = list(results.rows()) # Materialize results # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data else: downloadable = True data = [] results = type('Result', (object,), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) log = db.get_log(handle) except Exception, ex: fetch_error = True error_message, log = expand_exception(ex, db)
def execute_query(request, design_id=None): """ View function for executing an arbitrary query. It understands the optional GET/POST params: on_success_url If given, it will be displayed when the query is successfully finished. Otherwise, it will display the view query results page by default. """ authorized_get_design(request, design_id) error_message = None form = QueryForm() action = request.path log = None design = safe_get_design(request, models.SavedQuery.HQL, design_id) on_success_url = request.REQUEST.get('on_success_url') if request.method == 'POST': form.bind(request.POST) to_explain = request.POST.has_key('button-explain') to_submit = request.POST.has_key('button-submit') # Always validate the saveform, which will tell us whether it needs explicit saving if form.is_valid(): to_save = form.saveform.cleaned_data['save'] to_saveas = form.saveform.cleaned_data['saveas'] if to_saveas and not design.is_auto: # Save As only affects a previously saved query design = design.clone() if to_submit or to_save or to_saveas or to_explain: explicit_save = to_save or to_saveas design = save_design(request, form, models.SavedQuery.HQL, design, explicit_save) action = urlresolvers.reverse(execute_query, kwargs=dict(design_id=design.id)) if to_explain or to_submit: query_str = form.query.cleaned_data["query"] query_server = get_query_server(form.query_servers.cleaned_data["server"]) # (Optional) Parameterization. parameterization = get_parameterization(request, query_str, form, design, to_explain) if parameterization: return parameterization try: query = HQLdesign(form) if to_explain: return explain_directly(request, query, design, query_server) else: notify = form.query.cleaned_data.get('email_notify', False) return execute_directly(request, query, query_server, design, on_success_url=on_success_url, notify=notify) except BeeswaxException, ex: print ex.errorCode print ex.SQLState db = dbms.get(request.user, query_server) error_message, log = expand_exception(ex, db)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the watch_query view. If the result set has has_result_set=False, display an empty result. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See watch_query().) """ first_row = long(first_row) start_over = first_row == 0 results = type("Result", (object,), {"rows": 0, "columns": [], "has_more": False, "start_row": 0}) data = [] fetch_error = False error_message = "" log = "" app_name = get_app_name(request) query_history = authorized_get_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() db = dbms.get(request.user, query_server) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get("context", "") query_context = _parse_query_context(context_param) # To remove in Hue 2.3 download = request.GET.get("download", "") # Update the status as expired should not be accessible # Impala does not support startover for now expired = state == models.QueryHistory.STATE.expired if expired or app_name == "impala": state = models.QueryHistory.STATE.expired query_history.save_state(state) # Retrieve query results or use empty result if no result set try: if query_server["server_name"] == "impala" and not handle.has_result_set: downloadable = False elif not download: results = db.fetch(handle, start_over, 100) data = list(results.rows()) # Materialize results # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data log = db.get_log(handle) else: downloadable = True except Exception, ex: fetch_error = True error_message, log = expand_exception(ex, db, handle)
def _run_parameterized_query(request, design_id, explain): """ Given a design and arguments to parameterize that design, runs the query. - explain is a boolean to determine whether to run as an explain or as an execute. This is an extra "step" in the flow from execute_query. """ design = authorized_get_design(request, design_id, must_exist=True) # Reconstitute the form design_obj = beeswax.design.HQLdesign.loads(design.data) query_form = QueryForm() params = design_obj.get_query_dict() params.update(request.POST) databases = _get_db_choices(request) query_form.bind(params) query_form.query.fields["database"].choices = databases # Could not do it in the form if not query_form.is_valid(): raise PopupException(_("Query form is invalid: %s") % query_form.errors) query_str = query_form.query.cleaned_data["query"] app_name = get_app_name(request) query_server = get_query_server_config(app_name) query_type = SavedQuery.TYPES_MAPPING[app_name] parameterization_form_cls = make_parameterization_form(query_str) if not parameterization_form_cls: raise PopupException(_("Query is not parameterizable.")) parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization") if parameterization_form.is_valid(): real_query = substitute_variables(query_str, parameterization_form.cleaned_data) query = HQLdesign(query_form, query_type=query_type) query._data_dict["query"]["query"] = real_query try: if explain: return explain_directly(request, query, design, query_server) else: return execute_directly(request, query, query_server, design) except Exception, ex: db = dbms.get(request.user, query_server) error_message, log = expand_exception(ex, db) return render( "execute.mako", request, { "action": reverse(get_app_name(request) + ":execute_query"), "design": design, "error_message": error_message, "form": query_form, "log": log, "autocomplete_base_url": reverse(get_app_name(request) + ":autocomplete", kwargs={}), }, )
def save_results_hdfs_file(request, query_history_id): """ Save the results of a query to an HDFS file. Do not rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsFileForm({ 'target_file': request.POST.get('path'), 'overwrite': request.POST.get('overwrite', False), }) if form.is_valid(): target_file = form.cleaned_data['target_file'] overwrite = form.cleaned_data['overwrite'] try: handle, state = _get_query_handle_and_state(query_history) except Exception, ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: if overwrite and request.fs.exists(target_file): if request.fs.isfile(target_file): request.fs.do_as_user(request.user.username, request.fs.rmtree, target_file) else: raise PopupException(_("The target path is a directory")) upload(target_file, handle, request.user, db, request.fs) response['type'] = 'hdfs-file' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_file response['success_url'] = '/filebrowser/view=%s' % target_file response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3
def describe_table(request, database, table): db = dbms.get(request.user) error_message = '' table_data = '' table = db.get_table(database, table) try: table_data = db.get_sample(database, table) except BeeswaxException, ex: error_message, logs = expand_exception(ex, db)
def execute(request, design_id=None): response = {'status': -1, 'message': ''} if request.method != 'POST': response['message'] = _('A POST request is required.') app_name = get_app_name(request) query_server = get_query_server_config(app_name) query_type = beeswax.models.SavedQuery.TYPES_MAPPING[app_name] design = safe_get_design(request, query_type, design_id) try: query_form = get_query_form(request) if query_form.is_valid(): query_str = query_form.query.cleaned_data["query"] explain = request.GET.get('explain', 'false').lower() == 'true' design = save_design(request, query_form, query_type, design, False) if query_form.query.cleaned_data['is_parameterized']: # Parameterized query parameterization_form_cls = make_parameterization_form(query_str) if parameterization_form_cls: parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization") if parameterization_form.is_valid(): parameters = parameterization_form.cleaned_data real_query = substitute_variables(query_str, parameters) query = HQLdesign(query_form, query_type=query_type) query._data_dict['query']['query'] = real_query try: if explain: return explain_directly(request, query, design, query_server) else: return execute_directly(request, query, design, query_server, parameters=parameters) except Exception, ex: db = dbms.get(request.user, query_server) error_message, log = expand_exception(ex, db) response['message'] = error_message return JsonResponse(response) else: response['errors'] = parameterization_form.errors return JsonResponse(response) # Non-parameterized query query = HQLdesign(query_form, query_type=query_type) if request.GET.get('explain', 'false').lower() == 'true': return explain_directly(request, query, design, query_server) else: return execute_directly(request, query, design, query_server) else:
def describe_table(request, database, table): db = dbms.get(request.user) error_message = '' table_data = '' table = db.get_table(database, table) partitions = None if table.partition_keys: partitions = db.get_partitions(database, table, max_parts=None) try: table_data = db.get_sample(database, table) except Exception, ex: error_message, logs = dbms.expand_exception(ex, db)
def get_sample_data(request, database, table): db = dbms.get(request.user) response = {'status': -1, 'error_message': ''} try: table_obj = db.get_table(database, table) sample_data = db.get_sample(database, table_obj) response = { 'status': 0, 'headers': sample_data and sample_data.cols(), 'rows': sample_data and list(sample_data.rows()) } except Exception, ex: error_message, logs = dbms.expand_exception(ex, db) response['error_message'] = error_message
def drop_table(request, table): db = dbms.get(request.user) table = db.get_table('default', table) if request.method == 'POST': try: query_history = db.drop_table(table) url = reverse(watch_query, args=[query_history.id]) + '?on_success_url=' + urlresolvers.reverse(show_tables) return redirect(url) except BeeswaxException, ex: error_message, log = expand_exception(ex, db) error = _("Failed to remove %(table)s. Error: %(error)s") % {'table': table.name, 'error': error_message} raise PopupException(error, title=_("Beeswax Error"), detail=log)
def save_results_hive_table(request, query_history_id): """ Save the results of a query to a hive table. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) database = query_history.design.get_design().query.get('database', 'default') form = beeswax.forms.SaveResultsTableForm({ 'target_table': request.POST.get('table') }, db=db, database=database) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: query_history = db.create_table_as_a_select(request, query_history, form.target_database, form.cleaned_data['target_table'], result_meta) response['id'] = query_history.id response['query'] = query_history.query response['type'] = 'hive-table' response['path'] = form.cleaned_data['target_table'] response['success_url'] = reverse('metastore:describe_table', kwargs={'database': form.target_database, 'table': form.cleaned_data['target_table']}) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3
def drop_table(request, database): db = dbms.get(request.user) if request.method == 'POST': tables = request.POST.getlist('table_selection') tables_objects = [db.get_table(database, table) for table in tables] try: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user) query_history = db.drop_tables(database, tables_objects, design) url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('catalog:show_tables') return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message} raise PopupException(error, title=_("Beeswax Error"), detail=log)
def get_indexes(request, database, table): query_server = dbms.get_query_server_config(get_app_name(request)) db = dbms.get(request.user, query_server) response = {'status': -1, 'error_message': ''} try: indexes = db.get_indexes(database, table) if indexes: response['status'] = 0 response['headers'] = indexes.cols() response['rows'] = escape_rows(indexes.rows(), nulls_only=True) else: response['error_message'] = _('Index data took too long to be generated') except Exception, ex: error_message, logs = dbms.expand_exception(ex, db) response['error_message'] = error_message
def drop_partition(request, database, table): db = dbms.get(request.user) if request.method == 'POST': partition_specs = request.POST.getlist('partition_selection') partition_specs = [spec for spec in partition_specs] try: design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_partitions(database, table, partition_specs, design) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \ reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table}) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(partition)s. Error: %(error)s") % {'partition': '\n'.join(partition_specs), 'error': error_message} raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_database(request): db = dbms.get(request.user) if request.method == 'POST': databases = request.POST.getlist('database_selection') try: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_databases(databases, design) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases') return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message} raise PopupException(error, title=_("Hive Error"), detail=log)
def drop_table(request, database): db = dbms.get(request.user) if request.method == 'POST': tables = request.POST.getlist('table_selection') tables_objects = [db.get_table(database, table) for table in tables] try: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user) query_history = db.drop_tables(database, tables_objects, design) url = reverse('beeswax:watch_query', args=[query_history.id]) + '?on_success_url=' + reverse('metastore:show_tables') return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message} raise PopupException(error, title=_("Hive Error"), detail=log)
def get_sample_data(request, database, table): db = dbms.get(request.user) response = {'status': -1, 'error_message': ''} try: table_obj = db.get_table(database, table) sample_data = db.get_sample(database, table_obj) if sample_data: response['status'] = 0 response['headers'] = sample_data.cols() response['rows'] = escape_rows(sample_data.rows(), nulls_only=True) else: response['error_message'] = _('Sample data took too long to be generated') except Exception, ex: error_message, logs = dbms.expand_exception(ex, db) response['error_message'] = error_message
def _run_parameterized_query(request, design_id, explain): """ Given a design and arguments to parameterize that design, runs the query. - explain is a boolean to determine whether to run as an explain or as an execute. This is an extra "step" in the flow from execute_query. """ design = authorized_get_design(request, design_id, must_exist=True) # Reconstitute the form design_obj = beeswax.design.HQLdesign.loads(design.data) query_form = QueryForm() params = design_obj.get_query_dict() params.update(request.POST) query_form.bind(params) assert query_form.is_valid() query_str = query_form.query.cleaned_data["query"] query_server = get_query_server_config(get_app_name(request)) parameterization_form_cls = make_parameterization_form(query_str) if not parameterization_form_cls: raise PopupException(_("Query is not parameterizable.")) parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization") if parameterization_form.is_valid(): real_query = substitute_variables(query_str, parameterization_form.cleaned_data) query = HQLdesign(query_form) query._data_dict['query']['query'] = real_query try: if explain: return explain_directly(request, query, design, query_server) else: return execute_directly(request, query, query_server, design) except Exception, ex: db = dbms.get(request.user, query_server) error_message, log = expand_exception(ex, db) return render('execute.mako', request, { 'action': reverse(get_app_name(request) + ':execute_query'), 'design': design, 'error_message': error_message, 'form': query_form, 'log': log, 'autocomplete_base_url': reverse(get_app_name(request) + ':autocomplete', kwargs={}), })
def save_results(request, id): """ Save the results of a query to an HDFS directory or Hive table. """ query_history = authorized_get_history(request, id, must_exist=True) app_name = get_app_name(request) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method == "POST": if not query_history.is_success(): msg = _("This query is %(state)s. Results unavailable.") % {"state": state} raise PopupException(msg) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs) if request.POST.get("cancel"): return format_preserving_redirect(request, "/%s/watch/%s" % (app_name, id)) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: raise PopupException(_("Cannot find query: %s") % ex) try: if form.cleaned_data["save_target"] == form.SAVE_TYPE_DIR: target_dir = form.cleaned_data["target_dir"] query_history = db.insert_query_into_directory(query_history, target_dir) redirected = redirect( reverse("beeswax:watch_query", args=[query_history.id]) + "?on_success_url=" + reverse("filebrowser.views.view", kwargs={"path": target_dir}) ) elif form.cleaned_data["save_target"] == form.SAVE_TYPE_TBL: redirected = db.create_table_as_a_select( request, query_history, form.cleaned_data["target_table"], result_meta ) except Exception, ex: error_msg, log = expand_exception(ex, db) raise PopupException(_("The result could not be saved: %s.") % log, detail=ex) return redirected
def drop_database(request): source_type = request.POST.get('source_type', 'hive') db = _get_db(user=request.user, source_type=source_type) if request.method == 'POST': databases = request.POST.getlist('database_selection') try: if request.POST.get('is_embeddable'): design = SavedQuery.create_empty( app_name=source_type if source_type != 'hive' else 'beeswax', owner=request.user, data=hql_query('').dumps()) last_executed = json.loads(request.POST.get('start_time'), '-1') sql = db.drop_databases(databases, design, generate_ddl_only=True) job = make_notebook(name=_('Drop database %s') % ', '.join(databases)[:100], editor_type=source_type, statement=sql.strip(), status='ready', database=None, on_success_url='assist.db.refresh', is_task=True, last_executed=last_executed) return JsonResponse(job.execute(request)) else: design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_databases(databases, design) url = reverse( 'beeswax:watch_query_history', kwargs={ 'query_history_id': query_history.id }) + '?on_success_url=' + reverse('metastore:databases') return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(databases)s. Error: %(error)s") % { 'databases': ','.join(databases), 'error': error_message } raise PopupException(error, title=_("DB Error"), detail=log)
def drop_table(request, database): db = dbms.get(request.user) if request.method == 'POST': try: tables = request.POST.getlist('table_selection') tables_objects = [ db.get_table(database, table) for table in tables ] skip_trash = request.POST.get('skip_trash') == 'on' if request.POST.get('is_embeddable'): sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True) job = make_notebook(name='Execute and watch', editor_type='hive', statement=sql.strip(), status='ready', database=database, on_success_url='assist.db.refresh', is_task=True) return JsonResponse(job.execute(request)) else: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id }) + '?on_success_url=' + reverse( 'metastore:show_tables', kwargs={'database': database}) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(tables)s. Error: %(error)s") % { 'tables': ','.join(tables), 'error': error_message } raise PopupException(error, title=_("Hive Error"), detail=log)
def _run_parameterized_query(request, design_id, explain): """ Given a design and arguments to parameterize that design, runs the query. - explain is a boolean to determine whether to run as an explain or as an execute. This is an extra "step" in the flow from execute_query. """ design = authorized_get_design(request, design_id, must_exist=True) # Reconstitute the form design_obj = beeswax.design.HQLdesign.loads(design.data) query_form = QueryForm() params = design_obj.get_query_dict() params.update(request.POST) query_form.bind(params) assert query_form.is_valid() query_str = query_form.query.cleaned_data["query"] query_server = get_query_server_config(get_app_name(request)) parameterization_form_cls = make_parameterization_form(query_str) if not parameterization_form_cls: raise PopupException(_("Query is not parameterizable.")) parameterization_form = parameterization_form_cls(request.REQUEST, prefix="parameterization") if parameterization_form.is_valid(): real_query = substitute_variables(query_str, parameterization_form.cleaned_data) query = HQLdesign(query_form) query._data_dict['query']['query'] = real_query try: if explain: return explain_directly(request, query, design, query_server) else: return execute_directly(request, query, query_server, design) except Exception, ex: db = dbms.get(request.user, query_server) error_message, log = expand_exception(ex, db) return render('execute.mako', request, { 'action': reverse(get_app_name(request) + ':execute_query'), 'design': design, 'error_message': error_message, 'form': query_form, 'log': log, 'autocomplete': json.dumps({}), })
def drop_table(request, database): source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive')) cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, source_type=source_type, cluster=cluster) if request.method == 'POST': try: tables = request.POST.getlist('table_selection') tables_objects = [db.get_table(database, table) for table in tables] skip_trash = request.POST.get('skip_trash') == 'on' cluster = json.loads(request.POST.get('cluster', '{}')) namespace = json.loads(request.POST.get('namespace', '{}')) if request.POST.get('is_embeddable'): last_executed = json.loads(request.POST.get('start_time', '-1')) sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True) job = make_notebook( name=_('Drop table %s') % ', '.join([table.name for table in tables_objects])[:100], editor_type=source_type, statement=sql.strip(), status='ready', database=database, namespace=namespace, compute=cluster, on_success_url='assist.db.refresh', is_task=True, last_executed=last_executed ) return JsonResponse(job.execute(request)) else: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash) url = reverse( 'beeswax:watch_query_history', kwargs={'query_history_id': query_history.id} ) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database}) return redirect(url) except Exception as ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message} raise PopupException(error, title=_("DB Error"), detail=log) else: title = _("Do you really want to delete the table(s)?") return render('confirm.mako', request, {'url': request.path, 'title': title})
def save_results_hdfs_directory(request, query_history_id): """ Save the results of a query to an HDFS directory. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsDirectoryForm({ 'target_dir': request.POST.get('path') }, fs=request.fs) if form.is_valid(): target_dir = request.POST.get('path') try: response['type'] = 'hdfs-dir' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_dir response['success_url'] = '/filebrowser/view=%s' % target_dir query_history = db.insert_query_into_directory(query_history, target_dir) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception as ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3 else: response['status'] = 1 response['errors'] = form.errors return JsonResponse(response)
def drop_partition(request, database, table): source_type = request.POST.get('source_type', 'hive') cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, source_type=source_type, cluster=cluster) if request.method == 'POST': partition_specs = request.POST.getlist('partition_selection') partition_specs = [spec for spec in partition_specs] try: if request.GET.get("format", "html") == "json": last_executed = json.loads(request.POST.get('start_time'), '-1') sql = db.drop_partitions(database, table, partition_specs, design=None, generate_ddl_only=True) job = make_notebook(name=_('Drop partition %s') % ', '.join(partition_specs)[:100], editor_type=source_type, statement=sql.strip(), status='ready', database=None, on_success_url='assist.db.refresh', is_task=True, last_executed=last_executed) return JsonResponse(job.execute(request)) else: design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_partitions(database, table, partition_specs, design) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + \ reverse('metastore:describe_partitions', kwargs={'database': database, 'table': table}) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(partition)s. Error: %(error)s") % { 'partition': '\n'.join(partition_specs), 'error': error_message } raise PopupException(error, title=_("DB Error"), detail=log)
def save_results(request, id): """ DEPRECATED. Need to get rid of watch_wait dependency first. Save the results of a query to an HDFS directory or Hive table. """ query_history = authorized_get_history(request, id, must_exist=True) app_name = get_app_name(request) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method == 'POST': if not query_history.is_success(): msg = _('This query is %(state)s. Results unavailable.') % {'state': state} raise PopupException(msg) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs) if request.POST.get('cancel'): return format_preserving_redirect(request, '/%s/watch/%s' % (app_name, id)) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: raise PopupException(_('Cannot find query: %s') % ex) try: if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR: target_dir = form.cleaned_data['target_dir'] query_history = db.insert_query_into_directory(query_history, target_dir) redirected = redirect(reverse('beeswax:watch_query', args=[query_history.id]) \ + '?on_success_url=' + reverse('filebrowser.views.view', kwargs={'path': target_dir})) elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL: redirected = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta) except Exception, ex: error_msg, log = expand_exception(ex, db) raise PopupException(_('The result could not be saved: %s.') % log, detail=ex) return redirected
def save_results(request, id): """ Save the results of a query to an HDFS directory or Hive table. """ query_history = authorized_get_history(request, id, must_exist=True) app_name = get_app_name(request) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method == 'POST': if not query_history.is_success(): msg = _('This query is %(state)s. Results unavailable.') % {'state': state} raise PopupException(msg) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsForm(request.POST, db=db, fs=request.fs) if request.POST.get('cancel'): return format_preserving_redirect(request, '/%s/watch/%s' % (app_name, id)) if form.is_valid(): try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: raise PopupException(_('Cannot find query: %s') % ex) try: if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR: target_dir = form.cleaned_data['target_dir'] query_history = db.insert_query_into_directory(query_history, target_dir) redirected = redirect(reverse('beeswax:watch_query', args=[query_history.id]) \ + '?on_success_url=' + reverse('filebrowser.views.view', kwargs={'path': target_dir})) elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL: redirected = db.create_table_as_a_select(request, query_history, form.cleaned_data['target_table'], result_meta) except Exception, ex: error_msg, log = expand_exception(ex, db) raise PopupException(_('The result could not be saved: %s.') % log, detail=ex) return redirected
def save_results_hdfs_directory(request, query_history_id): """ Save the results of a query to an HDFS directory. Rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsDirectoryForm({ 'target_dir': request.POST.get('path') }, fs=request.fs) if form.is_valid(): target_dir = request.POST.get('path') try: response['type'] = 'hdfs-dir' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_dir response['success_url'] = '/filebrowser/view=%s' % target_dir query_history = db.insert_query_into_directory(query_history, target_dir) response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception, ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3 else:
def drop_table(request, database): source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive')) cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, source_type=source_type, cluster=cluster) if request.method == 'POST': try: tables = request.POST.getlist('table_selection') tables_objects = [db.get_table(database, table) for table in tables] skip_trash = request.POST.get('skip_trash') == 'on' cluster = json.loads(request.POST.get('cluster', '{}')) namespace = request.POST.get('namespace') if request.POST.get('is_embeddable'): last_executed = json.loads(request.POST.get('start_time'), '-1') sql = db.drop_tables(database, tables_objects, design=None, skip_trash=skip_trash, generate_ddl_only=True) job = make_notebook( name=_('Drop table %s') % ', '.join([table.name for table in tables_objects])[:100], editor_type=source_type, statement=sql.strip(), status='ready', database=database, namespace=namespace, compute=cluster, on_success_url='assist.db.refresh', is_task=True, last_executed=last_executed ) return JsonResponse(job.execute(request)) else: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_tables(database, tables_objects, design, skip_trash=skip_trash) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:show_tables', kwargs={'database': database}) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(tables)s. Error: %(error)s") % {'tables': ','.join(tables), 'error': error_message} raise PopupException(error, title=_("DB Error"), detail=log)
def drop_partition(request, database, table): db = dbms.get(request.user) if request.method == "POST": partition_specs = request.POST.getlist("partition_selection") partition_specs = [spec for spec in partition_specs] try: design = SavedQuery.create_empty(app_name="beeswax", owner=request.user, data=hql_query("").dumps()) query_history = db.drop_partitions(database, table, partition_specs, design) url = ( reverse("beeswax:watch_query_history", kwargs={"query_history_id": query_history.id}) + "?on_success_url=" + reverse("metastore:describe_partitions", kwargs={"database": database, "table": table}) ) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(partition)s. Error: %(error)s") % { "partition": "\n".join(partition_specs), "error": error_message, } raise PopupException(error, title=_("Hive Error"), detail=log)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the watch_query view. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See watch_query().) """ first_row = long(first_row) start_over = (first_row == 0) results = None data = None fetch_error = False error_message = '' log = '' query_history = authorized_get_history(request, id, must_exist=True) db = dbms.get(request.user, query_history.get_query_server()) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = _parse_query_context(context_param) # Retrieve query results try: results = db.fetch(handle, start_over, 100) data = list(results.rows()) # Materialize results # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data log = db.get_log(handle) except BeeswaxException, ex: fetch_error = True error_message, log = expand_exception(ex, db)
def drop_database(request): source_type = request.POST.get('source_type', request.GET.get('source_type', 'hive')) cluster = json.loads(request.POST.get('cluster', '{}')) db = _get_db(user=request.user, source_type=source_type, cluster=cluster) if request.method == 'POST': databases = request.POST.getlist('database_selection') try: if request.POST.get('is_embeddable'): design = SavedQuery.create_empty(app_name=source_type if source_type != 'hive' else 'beeswax', owner=request.user, data=hql_query('').dumps()) last_executed = json.loads(request.POST.get('start_time'), '-1') cluster = json.loads(request.POST.get('cluster', '{}')) namespace = request.POST.get('namespace') sql = db.drop_databases(databases, design, generate_ddl_only=True) job = make_notebook( name=_('Drop database %s') % ', '.join(databases)[:100], editor_type=source_type, statement=sql.strip(), status='ready', database=None, namespace=namespace, compute=cluster, on_success_url='assist.db.refresh', is_task=True, last_executed=last_executed ) return JsonResponse(job.execute(request)) else: design = SavedQuery.create_empty(app_name='beeswax', owner=request.user, data=hql_query('').dumps()) query_history = db.drop_databases(databases, design) url = reverse('beeswax:watch_query_history', kwargs={'query_history_id': query_history.id}) + '?on_success_url=' + reverse('metastore:databases') return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(databases)s. Error: %(error)s") % {'databases': ','.join(databases), 'error': error_message} raise PopupException(error, title=_("DB Error"), detail=log)
def drop_database(request): db = dbms.get(request.user) if request.method == "POST": databases = request.POST.getlist("database_selection") try: # Can't be simpler without an important refactoring design = SavedQuery.create_empty(app_name="beeswax", owner=request.user, data=hql_query("").dumps()) query_history = db.drop_databases(databases, design) url = ( reverse("beeswax:watch_query_history", kwargs={"query_history_id": query_history.id}) + "?on_success_url=" + reverse("metastore:databases") ) return redirect(url) except Exception, ex: error_message, log = dbms.expand_exception(ex, db) error = _("Failed to remove %(databases)s. Error: %(error)s") % { "databases": ",".join(databases), "error": error_message, } raise PopupException(error, title=_("Hive Error"), detail=log)
try: table = db.get_table(database, table) except Exception, e: if hasattr(e, 'message') and e.message: raise PopupException(_("Hive Error"), detail=e.message) else: raise PopupException(_("Hive Error"), detail=e) partitions = None if app_name != 'impala' and table.partition_keys: partitions = db.get_partitions(database, table, max_parts=None) try: table_data = db.get_sample(database, table) except Exception, ex: error_message, logs = dbms.expand_exception(ex, db) renderable = "describe_table.mako" if request.REQUEST.get("sample", "false") == "true": renderable = "sample.mako" return render( renderable, request, { 'breadcrumbs': [ { 'name': database, 'url': reverse('metastore:show_tables', kwargs={'database': database}) },
def save_results(request, id): """ Save the results of a query to an HDFS directory """ query_history = authorized_get_history(request, id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method == 'POST': # Make sure the result is available. # Note that we may still hit errors during the actual save if not query_history.is_success(): if query_history.is_failure(): msg = _('This query has %(state)s. Results unavailable.') % {'state': state} else: msg = _('The result of this query is not available yet.') raise PopupException(msg) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsForm(request.POST, db=db) # Cancel goes back to results if request.POST.get('cancel'): return format_preserving_redirect(request, '/beeswax/watch/%s' % (id,)) if form.is_valid(): # Do save # 1. Get the results metadata assert request.POST.get('save') try: handle, state = _get_query_handle_and_state(query_history) result_meta = db.get_results_metadata(handle) except Exception, ex: LOG.exception(ex) raise PopupException(_('Cannot find query.')) if result_meta.table_dir: result_meta.table_dir = request.fs.urlsplit(result_meta.table_dir)[2] # 2. Check for partitioned tables if result_meta.table_dir is None: raise PopupException(_('Saving results from a partitioned table is not supported. You may copy from the HDFS location manually.')) # 3. Actual saving of results try: if form.cleaned_data['save_target'] == form.SAVE_TYPE_DIR: # To dir if result_meta.in_tablename: raise PopupException(_('Saving results from a query with no MapReduce jobs is not supported. ' 'You may copy manually from the HDFS location %(path)s.') % {'path': result_meta.table_dir}) target_dir = form.cleaned_data['target_dir'] request.fs.rename_star(result_meta.table_dir, target_dir) LOG.debug("Moved results from %s to %s" % (result_meta.table_dir, target_dir)) query_history.save_state(models.QueryHistory.STATE.expired) return redirect(reverse('filebrowser.views.view', kwargs={'path': target_dir})) elif form.cleaned_data['save_target'] == form.SAVE_TYPE_TBL: # To new table try: return _save_results_ctas(request, query_history, form.cleaned_data['target_table'], result_meta) except Exception, bex: LOG.exception(bex) error_msg, log = expand_exception(bex, db) except WebHdfsException, ex: raise PopupException(_('The table could not be saved.'), detail=ex) except IOError, ex: LOG.exception(ex) error_msg = str(ex)
def execute_query(request, design_id=None): """ View function for executing an arbitrary query. It understands the optional GET/POST params: on_success_url If given, it will be displayed when the query is successfully finished. Otherwise, it will display the view query results page by default. """ authorized_get_design(request, design_id) error_message = None form = QueryForm() action = request.path log = None app_name = get_app_name(request) query_type = SavedQuery.TYPES_MAPPING[app_name] design = safe_get_design(request, query_type, design_id) on_success_url = request.REQUEST.get('on_success_url') query_server = get_query_server_config(app_name) db = dbms.get(request.user, query_server) databases = _get_db_choices(request) if request.method == 'POST': form.bind(request.POST) form.query.fields['database'].choices = databases # Could not do it in the form to_explain = request.POST.has_key('button-explain') to_submit = request.POST.has_key('button-submit') # Always validate the saveform, which will tell us whether it needs explicit saving if form.is_valid(): to_save = form.saveform.cleaned_data['save'] to_saveas = form.saveform.cleaned_data['saveas'] if to_saveas and not design.is_auto: # Save As only affects a previously saved query design = design.clone() if to_submit or to_save or to_saveas or to_explain: explicit_save = to_save or to_saveas design = save_design(request, form, query_type, design, explicit_save) action = reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id)) if to_explain or to_submit: query_str = form.query.cleaned_data["query"] # (Optional) Parameterization. parameterization = get_parameterization(request, query_str, form, design, to_explain) if parameterization: return parameterization try: query = HQLdesign(form, query_type=query_type) if to_explain: return explain_directly(request, query, design, query_server) else: download = request.POST.has_key('download') return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download) except Exception, ex: error_message, log = expand_exception(ex, db)
def save_results_hdfs_file(request, query_history_id): """ Save the results of a query to an HDFS file. Do not rerun the query. """ response = {'status': 0, 'message': ''} query_history = authorized_get_query_history(request, query_history_id, must_exist=True) server_id, state = _get_query_handle_and_state(query_history) query_history.save_state(state) error_msg, log = None, None if request.method != 'POST': response['message'] = _('A POST request is required.') else: if not query_history.is_success(): response['message'] = _('This query is %(state)s. Results unavailable.') % {'state': state} response['status'] = -1 return JsonResponse(response) db = dbms.get(request.user, query_history.get_query_server_config()) form = beeswax.forms.SaveResultsFileForm({ 'target_file': request.POST.get('path'), 'overwrite': request.POST.get('overwrite', False), }) if form.is_valid(): target_file = form.cleaned_data['target_file'] overwrite = form.cleaned_data['overwrite'] try: handle, state = _get_query_handle_and_state(query_history) except Exception as ex: response['message'] = _('Cannot find query handle and state: %s') % str(query_history) response['status'] = -2 return JsonResponse(response) try: if overwrite and request.fs.exists(target_file): if request.fs.isfile(target_file): request.fs.do_as_user(request.user.username, request.fs.rmtree, target_file) else: raise PopupException(_("The target path is a directory")) upload(target_file, handle, request.user, db, request.fs) response['type'] = 'hdfs-file' response['id'] = query_history.id response['query'] = query_history.query response['path'] = target_file response['success_url'] = '/filebrowser/view=%s' % target_file response['watch_url'] = reverse(get_app_name(request) + ':api_watch_query_refresh_json', kwargs={'id': query_history.id}) except Exception as ex: error_msg, log = expand_exception(ex, db) response['message'] = _('The result could not be saved: %s.') % error_msg response['status'] = -3 else: response['status'] = 1 response['errors'] = form.errors return JsonResponse(response)
def execute(request, design_id=None): response = {'status': -1, 'message': ''} if request.method != 'POST': response['message'] = _('A POST request is required.') app_name = get_app_name(request) query_server = get_query_server_config(app_name) query_type = beeswax.models.SavedQuery.TYPES_MAPPING[app_name] design = safe_get_design(request, query_type, design_id) try: query_form = get_query_form(request) if query_form.is_valid(): query_str = query_form.query.cleaned_data["query"] explain = request.GET.get('explain', 'false').lower() == 'true' design = save_design(request, query_form, query_type, design, False) if query_form.query.cleaned_data['is_parameterized']: # Parameterized query parameterization_form_cls = make_parameterization_form(query_str) if parameterization_form_cls: parameterization_form = parameterization_form_cls(request.POST.get('query-query', ''), prefix="parameterization") if parameterization_form.is_valid(): parameters = parameterization_form.cleaned_data real_query = substitute_variables(query_str, parameters) query = HQLdesign(query_form, query_type=query_type) query._data_dict['query']['query'] = real_query try: if explain: return explain_directly(request, query_server, query) else: return execute_directly(request, query, design, query_server, parameters=parameters) except Exception as ex: db = dbms.get(request.user, query_server) error_message, log = expand_exception(ex, db) response['message'] = error_message return JsonResponse(response) else: response['errors'] = parameterization_form.errors return JsonResponse(response) # Non-parameterized query query = HQLdesign(query_form, query_type=query_type) if request.GET.get('explain', 'false').lower() == 'true': return explain_directly(request, query_server, query) else: return execute_directly(request, query, design, query_server) else: response['message'] = _('There was an error with your query.') response['errors'] = { 'query': [query_form.query.errors], 'settings': query_form.settings.errors, 'file_resources': query_form.file_resources.errors, 'functions': query_form.functions.errors, } except RuntimeError as e: response['message']= str(e) return JsonResponse(response)
query_str = form.query.cleaned_data["query"] # (Optional) Parameterization. parameterization = get_parameterization(request, query_str, form, design, to_explain) if parameterization: return parameterization try: query = HQLdesign(form, query_type=query_type) if to_explain: return explain_directly(request, query, design, query_server) else: download = request.POST.has_key('download') return execute_directly(request, query, query_server, design, on_success_url=on_success_url, download=download) except Exception, ex: error_message, log = expand_exception(ex, db) else: if design.id is not None: data = HQLdesign.loads(design.data).get_query_dict() form.bind(data) form.saveform.set_data(design.name, design.desc) else: # New design form.bind() form.query.fields['database'].choices = databases # Could not do it in the form if not databases: request.error(_('No databases are available. Permissions could be missing.')) return render('execute.mako', request, { 'action': action,
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the watch_query view. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See watch_query().) """ first_row = long(first_row) start_over = (first_row == 0) results = None data = None fetch_error = False error_message = '' log = '' app_name = get_app_name(request) query_history = authorized_get_history(request, id, must_exist=True) db = dbms.get(request.user, query_history.get_query_server_config()) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = _parse_query_context(context_param) # To remove in Hue 2.3 download = request.GET.get('download', '') # Update the status as expired should not be accessible expired = state == QueryHistory.STATE.expired if expired: state = QueryHistory.STATE.expired query_history.save_state(state) # Retrieve query results try: if not download: results = db.fetch(handle, start_over, 100) data = list(results.rows()) # Materialize results # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data else: downloadable = True data = [] results = type('Result', (object, ), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) log = db.get_log(handle) except Exception as ex: fetch_error = True error_message, log = expand_exception(ex, db) # Handle errors error = fetch_error or results is None or expired context = { 'error': error, 'error_message': error_message, 'has_more': True, 'query': query_history, 'results': data, 'expected_first_row': first_row, 'log': log, 'hadoop_jobs': _parse_out_hadoop_jobs(log)[0], 'query_context': query_context, 'can_save': False, 'context_param': context_param, 'expired': expired, 'app_name': app_name, 'download': download, } if not error: download_urls = {} if downloadable: for format in common.DL_FORMATS: download_urls[format] = urlresolvers.reverse( 'beeswax' + ':download', kwargs=dict(id=str(id), format=format)) save_form = SaveResultsForm() results.start_row = first_row context.update({ 'results': data, 'has_more': results.has_more, 'next_row': results.start_row + len(data), 'start_row': results.start_row, 'expected_first_row': first_row, 'columns': results.columns, 'download_urls': download_urls, 'save_form': save_form, 'can_save': query_history.owner == request.user and not download, }) return render('watch_results.mako', request, context)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the execute_query view. If the result set has has_result_set=False, display an empty result. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See execute_query().) """ first_row = long(first_row) start_over = (first_row == 0) results = type('Result', (object,), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) data = [] fetch_error = False error_message = '' log = '' columns = [] app_name = get_app_name(request) query_history = authorized_get_query_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() db = dbms.get(request.user, query_server) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = parse_query_context(context_param) # Update the status as expired should not be accessible expired = state == models.QueryHistory.STATE.expired # Retrieve query results or use empty result if no result set try: if query_server['server_name'] == 'impala' and not handle.has_result_set: downloadable = False else: results = db.fetch(handle, start_over, 100) data = [] # Materialize and HTML escape results # TODO: use Number + list comprehension for row in results.rows(): escaped_row = [] for field in row: if isinstance(field, (int, long, float, complex, bool)): if math.isnan(field) or math.isinf(field): escaped_field = json.dumps(field) else: escaped_field = field elif field is None: escaped_field = 'NULL' else: field = smart_unicode(field, errors='replace') # Prevent error when getting back non utf8 like charset=iso-8859-1 escaped_field = escape(field).replace(' ', ' ') escaped_row.append(escaped_field) data.append(escaped_row) # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data log = db.get_log(handle) columns = results.data_table.cols() except Exception, ex: fetch_error = True error_message, log = expand_exception(ex, db, handle)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the execute_query view. If the result set has has_result_set=False, display an empty result. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See execute_query().) """ first_row = long(first_row) start_over = (first_row == 0) results = type('Result', (object, ), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) data = [] fetch_error = False error_message = '' log = '' columns = [] app_name = get_app_name(request) query_history = authorized_get_query_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() db = dbms.get(request.user, query_server) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = parse_query_context(context_param) # Update the status as expired should not be accessible expired = state == models.QueryHistory.STATE.expired # Retrieve query results or use empty result if no result set try: if query_server[ 'server_name'] == 'impala' and not handle.has_result_set: downloadable = False else: results = db.fetch(handle, start_over, 100) # Materialize and HTML escape results data = escape_rows(results.rows()) # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data log = db.get_log(handle) columns = results.data_table.cols() except Exception, ex: LOG.exception('error fetching results') fetch_error = True error_message, log = expand_exception(ex, db, handle)
def view_results(request, id, first_row=0): """ Returns the view for the results of the QueryHistory with the given id. The query results MUST be ready. To display query results, one should always go through the execute_query view. If the result set has has_result_set=False, display an empty result. If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just spits out a warning if first_row doesn't match the servers conception. Multiple readers will produce a confusing interaction here, and that's known. It understands the ``context`` GET parameter. (See execute_query().) """ first_row = int(first_row) start_over = (first_row == 0) results = type('Result', (object, ), { 'rows': 0, 'columns': [], 'has_more': False, 'start_row': 0, }) data = [] fetch_error = False error_message = '' log = '' columns = [] app_name = get_app_name(request) query_history = authorized_get_query_history(request, id, must_exist=True) query_server = query_history.get_query_server_config() db = dbms.get(request.user, query_server) handle, state = _get_query_handle_and_state(query_history) context_param = request.GET.get('context', '') query_context = parse_query_context(context_param) # Update the status as expired should not be accessible expired = state == models.QueryHistory.STATE.expired # Retrieve query results or use empty result if no result set try: if query_server[ 'server_name'] == 'impala' and not handle.has_result_set: downloadable = False else: results = db.fetch(handle, start_over, 100) # Materialize and HTML escape results data = escape_rows(results.rows()) # We display the "Download" button only when we know that there are results: downloadable = first_row > 0 or data log = db.get_log(handle) columns = results.data_table.cols() except Exception as ex: LOG.exception('error fetching results') fetch_error = True error_message, log = expand_exception(ex, db, handle) # Handle errors error = fetch_error or results is None or expired context = { 'error': error, 'message': error_message, 'query': query_history, 'results': data, 'columns': columns, 'expected_first_row': first_row, 'log': log, 'hadoop_jobs': app_name != 'impala' and parse_out_jobs(log), 'query_context': query_context, 'can_save': False, 'context_param': context_param, 'expired': expired, 'app_name': app_name, 'next_json_set': None, 'is_finished': query_history.is_finished() } if not error: download_urls = {} if downloadable: for format in common.DL_FORMATS: download_urls[format] = reverse(app_name + ':download', kwargs=dict(id=str(id), format=format)) results.start_row = first_row context.update({ 'id': id, 'results': data, 'has_more': results.has_more, 'next_row': results.start_row + len(data), 'start_row': results.start_row, 'expected_first_row': first_row, 'columns': columns, 'download_urls': download_urls, 'can_save': query_history.owner == request.user, 'next_json_set': reverse(get_app_name(request) + ':view_results', kwargs={ 'id': str(id), 'first_row': results.start_row + len(data) }) + ('?context=' + context_param or '') + '&format=json' }) context['columns'] = massage_columns_for_json(columns) if 'save_form' in context: del context['save_form'] if 'query' in context: del context['query'] return JsonResponse(context)
def execute_query(request, design_id=None): """ View function for executing an arbitrary query. It understands the optional GET/POST params: on_success_url If given, it will be displayed when the query is successfully finished. Otherwise, it will display the view query results page by default. """ authorized_get_design(request, design_id) request.session['start_time'] = time.time( ) # FIXME: add job id to not intersect simultaneous jobs error_message = None form = QueryForm() action = request.path log = None app_name = get_app_name(request) query_type = SavedQuery.TYPES_MAPPING['beeswax'] design = safe_get_design(request, query_type, design_id) on_success_url = request.REQUEST.get('on_success_url') query_server = get_query_server_config(app_name) db = dbms.get(request.user, query_server) databases = _get_db_choices(request) if request.method == 'POST': form.bind(request.POST) form.query.fields[ 'database'].choices = databases # Could not do it in the form to_explain = request.POST.has_key('button-explain') to_submit = request.POST.has_key('button-submit') # Always validate the saveform, which will tell us whether it needs explicit saving if form.is_valid(): to_save = form.saveform.cleaned_data['save'] to_saveas = form.saveform.cleaned_data['saveas'] if to_save or to_saveas: if 'beeswax-autosave' in request.session: del request.session['beeswax-autosave'] if to_saveas and not design.is_auto: # Save As only affects a previously saved query design = design.clone() if to_submit or to_save or to_saveas or to_explain: explicit_save = to_save or to_saveas design = save_design(request, form, query_type, design, explicit_save) action = urlresolvers.reverse(app_name + ':execute_query', kwargs=dict(design_id=design.id)) if to_explain or to_submit: query_str = form.query.cleaned_data["query"] if conf.CHECK_PARTITION_CLAUSE_IN_QUERY.get(): query_str = _strip_trailing_semicolon(query_str) # check query. if a select query on partitioned table without partition keys, # intercept it and raise a PopupException. _check_partition_clause_in_query( form.query.cleaned_data.get('database', None), query_str, db) # (Optional) Parameterization. parameterization = get_parameterization( request, query_str, form, design, to_explain) if parameterization: return parameterization try: query = HQLdesign(form, query_type=query_type) if to_explain: return explain_directly(request, query, design, query_server) else: download = request.POST.has_key('download') download_format = form.query.cleaned_data.get( 'download_format', None) if not download_format: download_format = None if download_format in common.DL_FORMATS: request.session['dl_status'] = True return execute_directly( request, query, query_server, design, on_success_url=on_success_url, download_format=download_format, download=download) except QueryServerException, ex: error_message, log = expand_exception(ex, db)