def list_oozie_workflow(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
oozie_coordinator = None
if request.GET.get('coordinator_job_id'):
oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id'))
oozie_bundle = None
if request.GET.get('bundle_job_id'):
oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id'))
if oozie_coordinator is not None:
setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator)
if oozie_bundle is not None:
setattr(oozie_workflow, 'oozie_bundle', oozie_bundle)
oozie_parent = oozie_workflow.get_parent_job_id()
if oozie_parent:
oozie_parent = check_job_access_permission(request, oozie_parent)
workflow_data = None
credentials = None
doc = None
hue_workflow = None
workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows
full_node_list = None
if ENABLE_V2.get():
try:
# To update with the new History document model
hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(oozie_workflow.conf_dict)
if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user)
if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user)
if hue_workflow:
workflow_graph = hue_workflow.gen_status_graph(oozie_workflow)
full_node_list = hue_workflow.nodes
workflow_id = hue_workflow.id
wid = {
'id': workflow_id
}
doc = Document2.objects.get(type='oozie-workflow2', **wid)
new_workflow = get_workflow()(document=doc)
workflow_data = new_workflow.get_data()
credentials = Credentials()
else:
try:
Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow)
except Exception, e:
LOG.exception(_('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)))
except:
LOG.exception("Ignoring error updating Document2 record for job_id=%s", job_id)
else:
history = get_history().cross_reference_submission_history(request.user, job_id)
hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict)
if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id)
if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id)
if hue_workflow:
workflow_graph = hue_workflow.gen_status_graph(oozie_workflow)
full_node_list = hue_workflow.node_list
else:
workflow_graph, full_node_list = get_workflow().gen_status_graph_from_xml(request.user, oozie_workflow)
parameters = oozie_workflow.conf_dict.copy()
for action in oozie_workflow.actions:
action.oozie_coordinator = oozie_coordinator
action.oozie_bundle = oozie_bundle
if request.GET.get('format') == 'json':
return_obj = {
'id': oozie_workflow.id,
'status': oozie_workflow.status,
'progress': oozie_workflow.get_progress(full_node_list),
'graph': workflow_graph,
'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle)
}
return JsonResponse(return_obj, encoder=JSONEncoderForHTML)
oozie_slas = []
if oozie_workflow.has_sla:
oozie_api = get_oozie(request.user, api_version="v2")
params = {
'id': oozie_workflow.id,
'parent_id': oozie_workflow.id
}
oozie_slas = oozie_api.get_oozie_slas(**params)
return render('dashboard/list_oozie_workflow.mako', request, {
'oozie_workflow': oozie_workflow,
'oozie_coordinator': oozie_coordinator,
'oozie_bundle': oozie_bundle,
'oozie_parent': oozie_parent,
'oozie_slas': oozie_slas,
'hue_workflow': hue_workflow,
'hue_coord': hue_coord,
'parameters': parameters,
'has_job_edition_permission': has_job_edition_permission,
'workflow_graph': workflow_graph,
'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML) if workflow_data else '',
'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML) if workflow_data else '',
'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML) if credentials else '',
'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc1_id': doc.doc.get().id if doc else -1,
'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user))
})
'oldLogsApi': USE_GET_LOG_API.get()
}
# Run time error
if query_history.is_failure():
res = db.get_operation_status(handle)
if query_history.is_canceled(res):
result['status'] = 0
elif hasattr(res, 'errorMessage') and res.errorMessage:
result['message'] = res.errorMessage
else:
result['message'] = _('Bad status for request %s:\n%s') % (id, res)
else:
result['status'] = 0
return JsonResponse(result)
def massage_job_urls_for_json(jobs):
massaged_jobs = []
for job in jobs:
massaged_jobs.append({
'name': job,
'url': reverse('jobbrowser.views.single_job', kwargs={'job': job})
})
return massaged_jobs
@error_handler
def close_operation(request, query_history_id):
response = {
def save_results_hive_table(request, query_history_id):
"""
Save the results of a query to a hive table.
Rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request,
query_history_id,
must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _(
'This query is %(state)s. Results unavailable.') % {
'state': state
}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
database = query_history.design.get_design().query.get(
'database', 'default')
form = beeswax.forms.SaveResultsTableForm(
{'target_table': request.POST.get('table')},
db=db,
database=database)
if form.is_valid():
try:
handle, state = _get_query_handle_and_state(query_history)
result_meta = db.get_results_metadata(handle)
except Exception, ex:
response['message'] = _(
'Cannot find query handle and state: %s') % str(
query_history)
response['status'] = -2
return JsonResponse(response)
try:
query_history = db.create_table_as_a_select(
request, query_history, form.target_database,
form.cleaned_data['target_table'], result_meta)
response['id'] = query_history.id
response['query'] = query_history.query
response['type'] = 'hive-table'
response['path'] = form.cleaned_data['target_table']
response['success_url'] = reverse(
'metastore:describe_table',
kwargs={
'database': form.target_database,
'table': form.cleaned_data['target_table']
})
response['watch_url'] = reverse(
get_app_name(request) + ':api_watch_query_refresh_json',
kwargs={'id': query_history.id})
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _(
'The result could not be saved: %s.') % error_msg
response['status'] = -3
format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n"}
elif file_format['inputFormat'] == 'connector':
if file_format['connectorSelection'] == 'sfdc':
sf = Salesforce(
username=file_format['streamUsername'],
password=file_format['streamPassword'],
security_token=file_format['streamToken']
)
format_ = {"type": "csv", "fieldSeparator": ",", "hasHeader": True, "quoteChar": "\"", "recordSeparator": "\\n", 'objects': [sobject['name'] for sobject in sf.restful('sobjects/')['sobjects'] if sobject['queryable']]}
else:
raise PopupException(_('Input format %(inputFormat)s connector not recognized: $(connectorSelection)s') % file_format)
else:
raise PopupException(_('Input format not recognized: %(inputFormat)s') % file_format)
format_['status'] = 0
return JsonResponse(format_)
def guess_field_types(request):
file_format = json.loads(request.POST.get('fileFormat', '{}'))
if file_format['inputFormat'] == 'file':
indexer = MorphlineIndexer(request.user, request.fs)
path = urllib.unquote(file_format["path"])
stream = request.fs.open(path)
encoding = chardet.detect(stream.read(10000)).get('encoding')
stream.seek(0)
_convert_format(file_format["format"], inverse=True)
format_ = indexer.guess_field_types({
"file": {
def describe_partitions(request, database, table):
db = _get_db(user=request.user)
table_obj = db.get_table(database, table)
if not table_obj.partition_keys:
raise PopupException(
_("Table '%(table)s' is not partitioned.") % {'table': table})
reverse_sort = request.GET.get("sort", "desc").lower() == "desc"
if request.method == "POST":
partition_filters = {}
for part in table_obj.partition_keys:
if request.GET.get(part.name):
partition_filters[part.name] = request.GET.get(part.name)
partition_spec = ','.join(
["%s='%s'" % (k, v) for k, v in partition_filters.items()])
else:
partition_spec = ''
try:
partitions = db.get_partitions(database,
table_obj,
partition_spec,
reverse_sort=reverse_sort)
except:
LOG.exception('Table partitions could not be retrieved')
partitions = []
massaged_partitions = [
_massage_partition(database, table_obj, partition)
for partition in partitions
]
if request.method == "POST" or request.GET.get('format', 'html') == 'json':
return JsonResponse({
'partition_keys_json':
[partition.name for partition in table_obj.partition_keys],
'partition_values_json':
massaged_partitions,
})
else:
return render(
"metastore.mako", request, {
'breadcrumbs': [
{
'name':
database,
'url':
reverse('metastore:show_tables',
kwargs={'database': database})
},
{
'name':
table,
'url':
reverse('metastore:describe_table',
kwargs={
'database': database,
'table': table
})
},
{
'name':
'partitions',
'url':
reverse('metastore:describe_partitions',
kwargs={
'database': database,
'table': table
})
},
],
'database':
database,
'table':
table_obj,
'partitions':
partitions,
'partition_keys_json':
json.dumps(
[partition.name
for partition in table_obj.partition_keys]),
'partition_values_json':
json.dumps(massaged_partitions),
'request':
request,
'has_write_access':
has_write_access(request.user),
'is_optimizer_enabled':
has_optimizer(),
'is_navigator_enabled':
has_navigator(request.user),
'optimizer_url':
get_optimizer_url(),
'navigator_url':
get_navigator_url(),
'is_embeddable':
request.GET.get('is_embeddable', False),
'source_type':
_get_servername(db),
})
def get_connectors_instances(request):
return JsonResponse(
{'connectors': _group_by_category(_get_installed_connectors())})
return wraps(view_func)(decorate)
def job_not_assigned(request, jobid, path):
if request.GET.get('format') == 'json':
result = {'status': -1, 'message': ''}
try:
get_api(request.user, request.jt).get_job(jobid=jobid)
result['status'] = 0
except ApplicationNotRunning, e:
result['status'] = 1
except Exception, e:
result['message'] = _('Error polling job %s: %s') % (jobid, e)
return JsonResponse(result, encoder=JSONEncoderForHTML)
else:
return render('job_not_assigned.mako', request, {
'jobid': jobid,
'path': path
})
def jobs(request):
user = request.GET.get('user', request.user.username)
state = request.GET.get('state')
text = request.GET.get('text')
retired = request.GET.get('retired')
if request.GET.get('format') == 'json':
try:
def dt_login(request, from_modal=False):
if request.method == 'GET':
redirect_to = request.GET.get('next', '/')
else:
redirect_to = request.POST.get('next', '/')
is_first_login_ever = first_login_ever()
backend_names = auth_forms.get_backend_names()
is_active_directory = auth_forms.is_active_directory()
is_ldap_option_selected = 'server' not in request.POST or request.POST.get('server') == 'LDAP' \
or request.POST.get('server') in auth_forms.get_ldap_server_keys()
if is_active_directory and is_ldap_option_selected:
UserCreationForm = auth_forms.LdapUserCreationForm
AuthenticationForm = auth_forms.LdapAuthenticationForm
else:
UserCreationForm = auth_forms.UserCreationForm
if 'ImpersonationBackend' in backend_names:
AuthenticationForm = ImpersonationAuthenticationForm
else:
AuthenticationForm = auth_forms.AuthenticationForm
if request.method == 'POST':
request.audit = {
'operation': 'USER_LOGIN',
'username': request.POST.get('username')
}
# For first login, need to validate user info!
first_user_form = is_first_login_ever and UserCreationForm(
data=request.POST) or None
first_user = first_user_form and first_user_form.is_valid()
if first_user or not is_first_login_ever:
auth_form = AuthenticationForm(data=request.POST)
if auth_form.is_valid():
# Must login by using the AuthenticationForm. It provides 'backends' on the User object.
user = auth_form.get_user()
userprofile = get_profile(user)
login(request, user)
if request.session.test_cookie_worked():
request.session.delete_test_cookie()
try:
ensure_home_directory(request.fs, user)
except (IOError, WebHdfsException) as e:
LOG.error(
'Could not create home directory at login for %s.' %
user,
exc_info=e)
if require_change_password(userprofile):
return HttpResponseRedirect(
urlresolvers.reverse(
'useradmin.views.edit_user',
kwargs={'username': user.username}))
userprofile.first_login = False
userprofile.last_activity = datetime.now()
# This is to fix a bug in Hue 4.3
if userprofile.creation_method == UserProfile.CreationMethod.EXTERNAL:
userprofile.creation_method = UserProfile.CreationMethod.EXTERNAL.name
userprofile.save()
msg = 'Successful login for user: %s' % user.username
request.audit['operationText'] = msg
access_warn(request, msg)
if from_modal or request.GET.get('fromModal',
'false') == 'true':
return JsonResponse({'auth': True})
else:
return HttpResponseRedirect(redirect_to)
else:
request.audit['allowed'] = False
msg = 'Failed login for user: %s' % request.POST.get(
'username')
request.audit['operationText'] = msg
access_warn(request, msg)
if from_modal or request.GET.get('fromModal',
'false') == 'true':
return JsonResponse({'auth': False})
else:
first_user_form = None
auth_form = AuthenticationForm()
# SAML/OIDC user is already authenticated in djangosaml2.views.login
if hasattr(request, 'fs') and (
'KnoxSpnegoDjangoBackend' in backend_names
or 'SpnegoDjangoBackend' in backend_names
or 'OIDCBackend' in backend_names or 'SAML2Backend'
in backend_names) and request.user.is_authenticated():
try:
ensure_home_directory(request.fs, request.user)
except (IOError, WebHdfsException) as e:
LOG.error(
'Could not create home directory for %s user %s.' %
('OIDC' if 'OIDCBackend' in backend_names else 'SAML',
request.user))
if request.user.is_authenticated():
return HttpResponseRedirect(redirect_to)
if is_active_directory and not is_ldap_option_selected and \
request.method == 'POST' and request.user.username != request.POST.get('username'):
# local user login failed, give the right auth_form with 'server' field
auth_form = auth_forms.LdapAuthenticationForm()
if not from_modal:
request.session.set_test_cookie()
renderable_path = 'login.mako'
if from_modal:
renderable_path = 'login_modal.mako'
response = render(
renderable_path, request, {
'action': urlresolvers.reverse('desktop_auth_views_dt_login'),
'form': first_user_form or auth_form,
'next': redirect_to,
'first_login_ever': is_first_login_ever,
'login_errors': request.method == 'POST',
'backend_names': backend_names,
'active_directory': is_active_directory,
'user': request.user
})
if not request.user.is_authenticated():
response.delete_cookie(
LOAD_BALANCER_COOKIE
) # Note: might be re-balanced to another Hue on login.
return response
def drop_table(request, database):
source_type = request.POST.get('source_type',
request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
if request.method == 'POST':
try:
tables = request.POST.getlist('table_selection')
tables_objects = [
db.get_table(database, table) for table in tables
]
skip_trash = request.POST.get('skip_trash') == 'on'
cluster = json.loads(request.POST.get('cluster', '{}'))
namespace = json.loads(request.POST.get('namespace', '{}'))
if request.POST.get('is_embeddable'):
last_executed = json.loads(request.POST.get(
'start_time', '-1'))
sql = db.drop_tables(database,
tables_objects,
design=None,
skip_trash=skip_trash,
generate_ddl_only=True)
job = make_notebook(
name=_('Drop table %s') %
', '.join([table.name for table in tables_objects])[:100],
editor_type=source_type,
statement=sql.strip(),
status='ready',
database=database,
namespace=namespace,
compute=cluster,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed)
return JsonResponse(job.execute(request))
else:
# Can't be simpler without an important refactoring
design = SavedQuery.create_empty(app_name='beeswax',
owner=request.user,
data=hql_query('').dumps())
query_history = db.drop_tables(database,
tables_objects,
design,
skip_trash=skip_trash)
url = reverse('beeswax:watch_query_history',
kwargs={'query_history_id': query_history.id
}) + '?on_success_url=' + reverse(
'metastore:show_tables',
kwargs={'database': database})
return redirect(url)
except Exception as ex:
error_message, log = dbms.expand_exception(ex, db)
error = _("Failed to remove %(tables)s. Error: %(error)s") % {
'tables': ','.join(tables),
'error': error_message
}
raise PopupException(error, title=_("DB Error"), detail=log)
else:
title = _("Do you really want to delete the table(s)?")
return render('confirm.mako', request, {
'url': request.path,
'title': title
})
def load_table(request, database, table):
response = {'status': -1, 'data': 'None'}
source_type = request.POST.get('source_type',
request.GET.get('source_type', 'hive'))
cluster = json.loads(request.POST.get('cluster', '{}'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
table = db.get_table(database, table)
if request.method == "POST":
load_form = LoadDataForm(table, request.POST)
if load_form.is_valid():
on_success_url = reverse('metastore:describe_table',
kwargs={
'database': database,
'table': table.name
})
generate_ddl_only = request.POST.get('is_embeddable',
'false') == 'true'
try:
design = SavedQuery.create_empty(
app_name=source_type
if source_type != 'hive' else 'beeswax',
owner=request.user,
data=hql_query('').dumps())
form_data = {
'path':
load_form.cleaned_data['path'],
'overwrite':
load_form.cleaned_data['overwrite'],
'partition_columns':
[(column_name, load_form.cleaned_data[key]) for key,
column_name in load_form.partition_columns.items()],
}
query_history = db.load_data(
database,
table.name,
form_data,
design,
generate_ddl_only=generate_ddl_only)
if generate_ddl_only:
last_executed = json.loads(request.POST.get('start_time'),
'-1')
job = make_notebook(name=_('Load data in %s.%s') %
(database, table.name),
editor_type=source_type,
statement=query_history.strip(),
status='ready',
database=database,
on_success_url='assist.db.refresh',
is_task=True,
last_executed=last_executed)
response = job.execute(request)
else:
url = reverse('beeswax:watch_query_history',
kwargs={
'query_history_id': query_history.id
}) + '?on_success_url=' + on_success_url
response['status'] = 0
response['data'] = url
response['query_history_id'] = query_history.id
except QueryError as ex:
response['status'] = 1
response['data'] = _("Can't load the data: ") + ex.message
except Exception as e:
response['status'] = 1
response['data'] = _("Can't load the data: ") + str(e)
else:
load_form = LoadDataForm(table)
if response['status'] == -1:
popup = render('popups/load_data.mako',
request, {
'table': table,
'load_form': load_form,
'source_type': source_type,
'database': database,
'app_name': 'beeswax'
},
force_template=True).content
response['data'] = popup
return JsonResponse(response)
def describe_table(request, database, table):
app_name = get_app_name(request)
cluster = json.loads(request.POST.get('cluster', '{}'))
source_type = request.POST.get('source_type',
request.GET.get('source_type', 'hive'))
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
try:
table = db.get_table(database, table)
except Exception as e:
LOG.exception("Describe table error")
raise PopupException(
_("DB Error"),
detail=e.message if hasattr(e, 'message') and e.message else e)
if request.POST.get("format", "html") == "json":
return JsonResponse({
'status':
0,
'name':
table.name,
'partition_keys': [{
'name': part.name,
'type': part.type
} for part in table.partition_keys],
'cols': [{
'name': col.name,
'type': col.type,
'comment': col.comment
} for col in table.cols],
'path_location':
table.path_location,
'hdfs_link':
table.hdfs_link,
'comment':
table.comment,
'is_view':
table.is_view,
'properties':
table.properties,
'details':
table.details,
'stats':
table.stats
})
else: # Render HTML
renderable = "metastore.mako"
apps_list = _get_apps(request.user, '')
partitions = None
if app_name != 'impala' and table.partition_keys:
try:
partitions = [
_massage_partition(database, table, partition)
for partition in db.get_partitions(database, table)
]
except:
LOG.exception('Table partitions could not be retrieved')
return render(
renderable, request, {
'apps':
apps_list,
'breadcrumbs': [
{
'name':
database,
'url':
reverse('metastore:show_tables',
kwargs={'database': database})
},
{
'name':
str(table.name),
'url':
reverse('metastore:describe_table',
kwargs={
'database': database,
'table': table.name
})
},
],
'table':
table,
'partitions':
partitions,
'database':
database,
'has_write_access':
has_write_access(request.user),
'is_optimizer_enabled':
has_optimizer(),
'is_navigator_enabled':
has_catalog(request.user),
'optimizer_url':
get_optimizer_url(),
'navigator_url':
get_catalog_url(),
'is_embeddable':
request.GET.get('is_embeddable', False),
'source_type':
_get_servername(db),
})
return view_fn(*args, **kwargs)
except Http404, e:
raise e
except NavOptException, e:
LOG.exception(e)
response = {'status': -1, 'message': e.message}
except MissingSentryPrivilegeException, e:
LOG.exception(e)
response = {
'status': -1,
'message': 'Missing privileges for %s' % force_unicode(str(e))
}
except Exception, e:
LOG.exception(e)
response = {'status': -1, 'message': force_unicode(str(e))}
return JsonResponse(response, status=500)
return decorator
@require_POST
@error_handler
def get_tenant(request):
response = {'status': -1}
email = request.POST.get('email')
api = OptimizerApi(request.user)
data = api.get_tenant(email=email)
if data:
def view_results(request, id, first_row=0):
"""
Returns the view for the results of the QueryHistory with the given id.
The query results MUST be ready.
To display query results, one should always go through the execute_query view.
If the result set has has_result_set=False, display an empty result.
If ``first_row`` is 0, restarts (if necessary) the query read. Otherwise, just
spits out a warning if first_row doesn't match the servers conception.
Multiple readers will produce a confusing interaction here, and that's known.
It understands the ``context`` GET parameter. (See execute_query().)
"""
first_row = int(first_row)
start_over = (first_row == 0)
results = type('Result', (object,), {
'rows': 0,
'columns': [],
'has_more': False,
'start_row': 0,
})
data = []
fetch_error = False
error_message = ''
log = ''
columns = []
app_name = get_app_name(request)
query_history = authorized_get_query_history(request, id, must_exist=True)
query_server = query_history.get_query_server_config()
db = dbms.get(request.user, query_server)
handle, state = _get_query_handle_and_state(query_history)
context_param = request.GET.get('context', '')
query_context = parse_query_context(context_param)
# Update the status as expired should not be accessible
expired = state == models.QueryHistory.STATE.expired
# Retrieve query results or use empty result if no result set
try:
if query_server['server_name'] == 'impala' and not handle.has_result_set:
downloadable = False
else:
results = db.fetch(handle, start_over, 100)
# Materialize and HTML escape results
data = escape_rows(results.rows())
# We display the "Download" button only when we know that there are results:
downloadable = first_row > 0 or data
log = db.get_log(handle)
columns = results.data_table.cols()
except Exception as ex:
LOG.exception('error fetching results')
fetch_error = True
error_message, log = expand_exception(ex, db, handle)
# Handle errors
error = fetch_error or results is None or expired
context = {
'error': error,
'message': error_message,
'query': query_history,
'results': data,
'columns': columns,
'expected_first_row': first_row,
'log': log,
'hadoop_jobs': app_name != 'impala' and parse_out_jobs(log),
'query_context': query_context,
'can_save': False,
'context_param': context_param,
'expired': expired,
'app_name': app_name,
'next_json_set': None,
'is_finished': query_history.is_finished()
}
if not error:
download_urls = {}
if downloadable:
for format in common.DL_FORMATS:
download_urls[format] = reverse(app_name + ':download', kwargs=dict(id=str(id), format=format))
results.start_row = first_row
context.update({
'id': id,
'results': data,
'has_more': results.has_more,
'next_row': results.start_row + len(data),
'start_row': results.start_row,
'expected_first_row': first_row,
'columns': columns,
'download_urls': download_urls,
'can_save': query_history.owner == request.user,
'next_json_set':
reverse(get_app_name(request) + ':view_results', kwargs={
'id': str(id),
'first_row': results.start_row + len(data)
}
)
+ ('?context=' + context_param or '') + '&format=json'
})
context['columns'] = massage_columns_for_json(columns)
if 'save_form' in context:
del context['save_form']
if 'query' in context:
del context['query']
return JsonResponse(context)
def submit_coordinator(request, doc_id):
# TODO: Replace URL by desktop/scheduler API
if doc_id.isdigit():
coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
else:
coordinator = Coordinator(document=Document2.objects.get_by_uuid(
user=request.user, uuid=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value'])
for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
jsonify = request.POST.get('format') == 'json'
try:
job_id = _submit_coordinator(request, coordinator, mapping)
except Exception as e:
message = force_unicode(str(e))
return JsonResponse({
'status': -1,
'message': message
},
safe=False)
if jsonify:
return JsonResponse(
{
'status': 0,
'job_id': job_id,
'type': 'schedule'
},
safe=False)
else:
request.info(_('Coordinator submitted.'))
return redirect(
reverse('oozie:list_oozie_coordinator',
kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' %
params_form.errors))
else:
parameters = coordinator.find_all_parameters()
initial_params = ParameterForm.get_initial_params(
dict([(param['name'], param['value']) for param in parameters]))
params_form = ParametersFormSet(initial=initial_params)
return render('/scheduler/submit_job_popup.mako',
request, {
'params_form':
params_form,
'name':
coordinator.name,
'action':
reverse('oozie:editor_submit_coordinator',
kwargs={'doc_id': coordinator.id}),
'show_dryrun':
True,
'return_json':
request.GET.get('format') == 'json'
},
force_template=True)
def single_spark_job(request, job):
if request.REQUEST.get('format') == 'json':
json_job = {'job': massage_job_for_json(job, request)}
return JsonResponse(json_job, encoder=JSONEncoderForHTML)
else:
return render('job.mako', request, {'request': request, 'job': job})
def get_context_namespaces(request, interface):
'''
Namespaces are node cluster contexts (e.g. Hive + Ranger) that can be queried by computes.
'''
response = {}
namespaces = []
clusters = list(get_clusters(request.user).values())
# Currently broken if not sent
namespaces.extend([{
'id': cluster['id'],
'name': cluster['name'],
'status': 'CREATED',
'computes': [cluster]
} for cluster in clusters if cluster.get('type') == 'direct'])
if interface == 'hive' or interface == 'impala' or interface == 'report':
if get_cluster_config(request.user)['has_computes']:
# Note: attaching computes to namespaces might be done via the frontend in the future
if interface == 'impala':
if IS_K8S_ONLY.get():
adb_clusters = DataWarehouse2Api(
request.user).list_clusters()['clusters']
else:
adb_clusters = AnalyticDbApi(
request.user).list_clusters()['clusters']
for _cluster in adb_clusters: # Add "fake" namespace if needed
if not _cluster.get('namespaceCrn'):
_cluster['namespaceCrn'] = _cluster['crn']
_cluster['id'] = _cluster['crn']
_cluster['namespaceName'] = _cluster['clusterName']
_cluster['name'] = _cluster['clusterName']
_cluster[
'compute_end_point'] = '%(publicHost)s' % _cluster[
'coordinatorEndpoint'] if IS_K8S_ONLY.get(
) else '',
else:
adb_clusters = []
if IS_K8S_ONLY.get():
sdx_namespaces = []
else:
sdx_namespaces = SdxApi(request.user).list_namespaces()
# Adding "fake" namespace for cluster without one
sdx_namespaces.extend([
_cluster for _cluster in adb_clusters
if not _cluster.get('namespaceCrn') or (
IS_K8S_ONLY.get() and 'TERMINAT' not in _cluster['status'])
])
namespaces.extend(
[{
'id':
namespace.get('crn', 'None'),
'name':
namespace.get('namespaceName'),
'status':
namespace.get('status'),
'computes': [
_cluster for _cluster in adb_clusters
if _cluster.get('namespaceCrn') == namespace.get('crn')
]
} for namespace in sdx_namespaces
if namespace.get('status') == 'CREATED' or IS_K8S_ONLY.get()])
response[interface] = namespaces
response['status'] = 0
return JsonResponse(response)
def importer_submit(request):
source = json.loads(request.POST.get('source', '{}'))
outputFormat = json.loads(request.POST.get('destination',
'{}'))['outputFormat']
destination = json.loads(request.POST.get('destination', '{}'))
destination['ouputFormat'] = outputFormat # Workaround a very weird bug
start_time = json.loads(request.POST.get('start_time', '-1'))
if source['inputFormat'] == 'file':
if source['path']:
path = urllib.unquote(source['path'])
source['path'] = request.fs.netnormpath(path)
parent_path = request.fs.parent_path(path)
stats = request.fs.stats(parent_path)
split = urlparse(path)
# Only for HDFS, import data and non-external table
if split.scheme in (
'', 'hdfs') and destination['importData'] and destination[
'useDefaultLocation'] and oct(
stats["mode"])[-1] != '7':
user_scratch_dir = request.fs.get_home_dir() + '/.scratchdir'
request.fs.do_as_user(request.user, request.fs.mkdir,
user_scratch_dir, 00777)
request.fs.do_as_user(request.user, request.fs.rename,
source['path'], user_scratch_dir)
source['path'] = user_scratch_dir + '/' + source['path'].split(
'/')[-1]
if destination['ouputFormat'] in ('database', 'table'):
destination['nonDefaultLocation'] = request.fs.netnormpath(
destination['nonDefaultLocation']) if destination[
'nonDefaultLocation'] else destination['nonDefaultLocation']
if destination['ouputFormat'] == 'index':
source['columns'] = destination['columns']
index_name = destination["name"]
if destination['indexerRunJob'] or source['inputFormat'] == 'stream':
_convert_format(source["format"], inverse=True)
job_handle = _large_indexing(
request,
source,
index_name,
start_time=start_time,
lib_path=destination['indexerJobLibPath'],
destination=destination)
else:
client = SolrClient(request.user)
job_handle = _small_indexing(request.user, request.fs, client,
source, destination, index_name)
elif source['inputFormat'] in (
'stream', 'sfdc') or destination['ouputFormat'] == 'stream':
job_handle = _envelope_job(request,
source,
destination,
start_time=start_time,
lib_path=destination['indexerJobLibPath'])
elif source['inputFormat'] == 'altus':
# BDR copy or DistCP + DDL + Sentry DDL copy
pass
elif source['inputFormat'] == 'rdbms':
if destination['outputFormat'] in ('database', 'file', 'table',
'hbase'):
job_handle = run_sqoop(request, source, destination, start_time)
elif destination['ouputFormat'] == 'database':
job_handle = _create_database(request, source, destination, start_time)
else:
job_handle = _create_table(request, source, destination, start_time)
request.audit = {
'operation': 'EXPORT',
'operationText':
'User %(username)s exported %(inputFormat)s to %(ouputFormat)s: %(name)s'
% {
'username': request.user.username,
'inputFormat': source['inputFormat'],
'ouputFormat': destination['ouputFormat'],
'name': destination['name'],
},
'allowed': True
}
return JsonResponse(job_handle)
_("Alter database requires a properties value of key-value pairs."
))
properties = json.loads(properties)
db.alter_database(database, properties=properties)
db_metadata = db.get_database(database)
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['status'] = 0
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Failed to alter database `%s`: %s") % (database,
ex)
return JsonResponse(response)
def get_database_metadata(request, database, cluster=None):
response = {'status': -1, 'data': ''}
source_type = request.POST.get('source_type', 'hive')
db = _get_db(user=request.user, source_type=source_type, cluster=cluster)
try:
db_metadata = db.get_database(database)
response['status'] = 0
db_metadata['hdfs_link'] = location_to_url(db_metadata['location'])
response['data'] = db_metadata
except Exception, ex:
response['status'] = 1
response['data'] = _("Cannot get metadata for database %s: %s") % (
LOG = logging.getLogger(__name__)
def api_error_handler(func):
def decorator(*args, **kwargs):
response = {}
try:
return func(*args, **kwargs)
except Exception, e:
LOG.exception('Error running %s' % func)
response['status'] = -1
response['message'] = force_unicode(str(e))
finally:
if response:
return JsonResponse(response)
return decorator
@api_error_handler
def search_documents(request):
"""
Returns the directories and documents based on given params that are accessible by the current user
Optional params:
perms=<mode> - Controls whether to retrieve owned, shared, or both. Defaults to both.
include_history=<bool> - Controls whether to retrieve history docs. Defaults to false.
flatten=<bool> - Controls whether to return documents in a flat list, or roll up documents to a common directory
if possible. Defaults to true.
page=<n> - Controls pagination. Defaults to 1.
limit=<n> - Controls limit per page. Defaults to all.
LOG = logging.getLogger(__name__)
def api_error_handler(func):
def decorator(*args, **kwargs):
response = {}
try:
return func(*args, **kwargs)
except Exception, e:
LOG.exception('Error running %s' % func)
response['status'] = -1
response['message'] = force_unicode(str(e))
finally:
if response:
return JsonResponse(response)
return decorator
@api_error_handler
def get_config(request):
cluster_config = ClusterConfig(request.user)
app_config = cluster_config.get_apps()
return JsonResponse({
'status':
0,
'app_config':
app_config,
'main_button_action':
def get_context_computes(request, interface):
'''
Some clusters like Snowball can have multiple computes for a certain languages (Hive, Impala...).
'''
response = {}
computes = []
clusters = list(get_clusters(request.user).values())
if get_cluster_config(
request.user
)['has_computes']: # TODO: only based on interface selected?
interpreter = get_interpreter(connector_type=interface,
user=request.user)
if interpreter['dialect'] == 'impala':
# dw_clusters = DataWarehouse2Api(request.user).list_clusters()['clusters']
dw_clusters = [
{
'crn': 'c1',
'clusterName': 'c1',
'status': 'created',
'options': {
'server_host': 'c1.gethue.com',
'server_port': 10000
}
},
{
'crn': 'c2',
'clusterName': 'c2',
'status': 'created',
'options': {
'server_host': 'c2.gethue.com',
'server_port': 10000
}
},
]
computes.extend([{
'id':
cluster.get('crn'),
'name':
cluster.get('clusterName'),
'status':
cluster.get('status'),
'namespace':
cluster.get('namespaceCrn', cluster.get('crn')),
'type':
interpreter['dialect'],
'options':
cluster['options'],
} for cluster in dw_clusters])
else:
# Currently broken if not sent
computes.extend([{
'id': cluster['id'],
'name': cluster['name'],
'namespace': cluster['id'],
'interface': interface,
'type': cluster['type'],
'options': {}
} for cluster in clusters if cluster.get('type') == 'direct'])
response[interface] = computes
response['status'] = 0
return JsonResponse(response)
def save_results_hdfs_file(request, query_history_id):
"""
Save the results of a query to an HDFS file.
Do not rerun the query.
"""
response = {'status': 0, 'message': ''}
query_history = authorized_get_query_history(request,
query_history_id,
must_exist=True)
server_id, state = _get_query_handle_and_state(query_history)
query_history.save_state(state)
error_msg, log = None, None
if request.method != 'POST':
response['message'] = _('A POST request is required.')
else:
if not query_history.is_success():
response['message'] = _(
'This query is %(state)s. Results unavailable.') % {
'state': state
}
response['status'] = -1
return JsonResponse(response)
db = dbms.get(request.user, query_history.get_query_server_config())
form = beeswax.forms.SaveResultsFileForm({
'target_file':
request.POST.get('path'),
'overwrite':
request.POST.get('overwrite', False),
})
if form.is_valid():
target_file = form.cleaned_data['target_file']
overwrite = form.cleaned_data['overwrite']
try:
handle, state = _get_query_handle_and_state(query_history)
except Exception, ex:
response['message'] = _(
'Cannot find query handle and state: %s') % str(
query_history)
response['status'] = -2
return JsonResponse(response)
try:
if overwrite and request.fs.exists(target_file):
if request.fs.isfile(target_file):
request.fs.do_as_user(request.user.username,
request.fs.rmtree, target_file)
else:
raise PopupException(
_("The target path is a directory"))
upload(target_file, handle, request.user, db, request.fs)
response['type'] = 'hdfs-file'
response['id'] = query_history.id
response['query'] = query_history.query
response['path'] = target_file
response['success_url'] = '/filebrowser/view%s' % target_file
response['watch_url'] = reverse(
get_app_name(request) + ':api_watch_query_refresh_json',
kwargs={'id': query_history.id})
except Exception, ex:
error_msg, log = expand_exception(ex, db)
response['message'] = _(
'The result could not be saved: %s.') % error_msg
response['status'] = -3
def copy_document(request):
uuid = json.loads(request.POST.get('uuid'), '""')
if not uuid:
raise PopupException(_('copy_document requires uuid'))
# Document2 and Document model objects are linked and both are saved when saving
document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
# Document model object
document1 = document.doc.get()
if document.type == 'directory':
raise PopupException(_('Directory copy is not supported'))
name = document.name + '-copy'
# Make the copy of the Document2 model object
copy_document = document.copy(name=name, owner=request.user)
# Make the copy of Document model object too
document1.copy(content_object=copy_document, name=name, owner=request.user)
# Import workspace for all oozie jobs
if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace
# Update the name field in the json 'data' field
if document.type == 'oozie-workflow2':
workflow = Workflow(document=document)
workflow.update_name(name)
workflow.update_uuid(copy_document.uuid)
_import_workspace(request.fs, request.user, workflow)
copy_document.update_data(
{'workflow': workflow.get_data()['workflow']})
copy_document.save()
if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
if document.type == 'oozie-bundle2':
bundle_or_coordinator = Bundle(document=document)
else:
bundle_or_coordinator = Coordinator(document=document)
json_data = bundle_or_coordinator.get_data_for_json()
json_data['name'] = name
json_data['uuid'] = copy_document.uuid
copy_document.update_data(json_data)
copy_document.save()
_import_workspace(request.fs, request.user, bundle_or_coordinator)
elif document.type == 'search-dashboard':
from dashboard.models import Collection2
collection = Collection2(request.user, document=document)
collection.data['collection']['label'] = name
collection.data['collection']['uuid'] = copy_document.uuid
copy_document.update_data(
{'collection': collection.data['collection']})
copy_document.save()
# Keep the document and data in sync
else:
copy_data = copy_document.data_dict
if 'name' in copy_data:
copy_data['name'] = name
if 'uuid' in copy_data:
copy_data['uuid'] = copy_document.uuid
copy_document.update_data(copy_data)
copy_document.save()
return JsonResponse({'status': 0, 'document': copy_document.to_dict()})
def process_view(self, request, view_func, view_args, view_kwargs):
"""
We also perform access logging in ``process_view()`` since we have the view function,
which tells us the log level. The downside is that we don't have the status code,
which isn't useful for status logging anyways.
"""
request.ts = time.time()
request.view_func = view_func
access_log_level = getattr(view_func, 'access_log_level', None)
# Skip loop for oidc
if request.path in [
'/oidc/authenticate/', '/oidc/callback/', '/oidc/logout/',
'/hue/oidc_failed/'
]:
return None
# Skip views not requiring login
# If the view has "opted out" of login required, skip
if hasattr(view_func, "login_notrequired"):
log_page_hit(request,
view_func,
level=access_log_level or logging.DEBUG)
return None
# There are certain django views which are also opt-out, but
# it would be evil to go add attributes to them
if view_func in DJANGO_VIEW_AUTH_WHITELIST:
log_page_hit(request,
view_func,
level=access_log_level or logging.DEBUG)
return None
# If user is logged in, check that he has permissions to access the app
if request.user.is_active and request.user.is_authenticated:
AppSpecificMiddleware.augment_request_with_app(request, view_func)
# Until Django 1.3 which resolves returning the URL name, just do a match of the name of the view
try:
access_view = 'access_view:%s:%s' % (
request._desktop_app, resolve(request.path)[0].__name__)
except Exception as e:
access_log(request,
'error checking view perm: %s' % e,
level=access_log_level)
access_view = ''
app_accessed = request._desktop_app
app_libs_whitelist = [
"desktop", "home", "home2", "about", "hue", "editor",
"notebook", "indexer", "404", "500", "403"
]
if has_connectors():
app_libs_whitelist.append('metadata')
if DASHBOARD_ENABLED.get():
app_libs_whitelist.append('dashboard')
# Accessing an app can access an underlying other app.
# e.g. impala or spark uses code from beeswax and so accessing impala shows up as beeswax here.
# Here we trust the URL to be the real app we need to check the perms.
ui_app_accessed = get_app_name(request)
if app_accessed != ui_app_accessed and ui_app_accessed not in (
'logs', 'accounts', 'login'):
app_accessed = ui_app_accessed
if app_accessed and \
app_accessed not in app_libs_whitelist and \
not (
is_admin(request.user) or
request.user.has_hue_permission(action="access", app=app_accessed) or
request.user.has_hue_permission(action=access_view, app=app_accessed)
) and \
not (app_accessed == '__debug__' and DJANGO_DEBUG_MODE.get()):
access_log(request,
'permission denied',
level=access_log_level)
return PopupException(_(
"You do not have permission to access the %(app_name)s application."
) % {
'app_name': app_accessed.capitalize()
},
error_code=401).response(request)
else:
if not hasattr(request, 'view_func'):
log_page_hit(request, view_func, level=access_log_level)
return None
if desktop.conf.CORS_ENABLED.get():
user = authenticate(request, username='******', password='******')
if user is not None:
login(request, user)
return None
logging.info("Redirecting to login page: %s", request.get_full_path())
access_log(request, 'login redirection', level=access_log_level)
no_idle_backends = ("libsaml.backend.SAML2Backend",
"desktop.auth.backend.SpnegoDjangoBackend",
"desktop.auth.backend.KnoxSpnegoDjangoBackend")
if request.ajax and all(no_idle_backend not in AUTH.BACKEND.get()
for no_idle_backend in no_idle_backends):
# Send back a magic header which causes Hue.Request to interpose itself
# in the ajax request and make the user login before resubmitting the
# request.
response = HttpResponse("/* login required */",
content_type="text/javascript")
response[MIDDLEWARE_HEADER] = 'LOGIN_REQUIRED'
return response
else:
if request.GET.get('is_embeddable'):
return JsonResponse(
{
'url':
"%s?%s=%s" %
(settings.LOGIN_URL, REDIRECT_FIELD_NAME,
quote('/hue' + request.get_full_path().replace(
'is_embeddable=true', '').replace('&&', '&')))
}
) # Remove embeddable so redirect from & to login works. Login page is not embeddable
else:
return HttpResponseRedirect(
"%s?%s=%s" % (settings.LOGIN_URL, REDIRECT_FIELD_NAME,
quote(request.get_full_path())))
def export_documents(request):
if request.GET.get('documents'):
selection = json.loads(request.GET.get('documents'))
else:
selection = json.loads(request.POST.get('documents'))
include_history = request.GET.get('history', 'false') == 'true'
# Only export documents the user has permissions to read
docs = Document2.objects.documents(user=request.user, perms='both', include_history=True, include_trashed=True).\
filter(id__in=selection).order_by('-id')
# Add any dependencies to the set of exported documents
export_doc_set = _get_dependencies(docs, include_history=include_history)
# For directories, add any children docs to the set of exported documents
export_doc_set.update(_get_dependencies(docs, deps_mode=False))
# Get PKs of documents to export
doc_ids = [doc.pk for doc in export_doc_set]
num_docs = len(doc_ids)
if len(selection) == 1 and num_docs >= len(selection) and docs[0].name:
filename = docs[0].name
else:
filename = 'hue-documents-%s-(%s)' % (
datetime.today().strftime('%Y-%m-%d'), num_docs)
f = string_io()
if doc_ids:
doc_ids = ','.join(map(str, doc_ids))
management.call_command('dumpdata',
'desktop.Document2',
primary_keys=doc_ids,
indent=2,
use_natural_foreign_keys=True,
verbosity=2,
stdout=f)
if request.GET.get('format') == 'json':
return JsonResponse(f.getvalue(), safe=False)
elif request.GET.get('format') == 'zip':
zfile = zipfile.ZipFile(f, 'w')
zfile.writestr("hue.json", f.getvalue())
for doc in docs:
if doc.type == 'notebook':
try:
from spark.models import Notebook
zfile.writestr("notebook-%s-%s.txt" % (doc.name, doc.id),
smart_str(Notebook(document=doc).get_str()))
except Exception as e:
LOG.exception(e)
zfile.close()
response = HttpResponse(content_type="application/zip")
response["Content-Length"] = len(f.getvalue())
response[
'Content-Disposition'] = 'attachment; filename="%s".zip' % filename
response.write(f.getvalue())
return response
else:
return make_response(f.getvalue(), 'json', filename)
response['data'] = oozie_api.job_control(job_id, action, parameters=params)
response['status'] = 0
if 'notification' in request.POST:
request.info(_(request.POST.get('notification')))
except RestException, ex:
ex_message = ex.message
if ex._headers.get('oozie-error-message'):
ex_message = ex._headers.get('oozie-error-message')
msg = "Error performing %s on Oozie job %s: %s." % (action, job_id, ex_message)
LOG.exception(msg)
response['data'] = _(msg)
return JsonResponse(response)
def bulk_manage_oozie_jobs(request):
if request.method != 'POST':
raise PopupException(_('Use a POST request to manage the Oozie jobs.'))
response = {'status': -1, 'data': ''}
if 'job_ids' in request.POST and 'action' in request.POST:
jobs = request.POST.get('job_ids').split()
response = {'totalRequests': len(jobs), 'totalErrors': 0, 'messages': ''}
oozie_api = get_oozie(request.user)
for job_id in jobs:
def get_config(request):
config = get_cluster_config(request.user)
config['clusters'] = list(get_clusters(request.user).values())
config['status'] = 0
return JsonResponse(config)
]:
try:
if snippet['status'] != 'running':
response['result'].append(
get_api(request, snippet).close_statement(snippet))
else:
LOG.info('Not closing SQL snippet as still running.')
except QueryExpired:
pass
except Exception, e:
LOG.exception('Error closing statement %s' % str(e))
response['status'] = 0
response['message'] = _('Notebook closed successfully')
return JsonResponse(response)
@require_POST
@check_document_access_permission()
def close_statement(request):
response = {'status': -1}
# Passed by check_document_access_permission but unused by APIs
notebook = json.loads(request.POST.get('notebook', '{}'))
snippet = json.loads(request.POST.get('snippet', '{}'))
try:
response['result'] = get_api(request, snippet).close_statement(snippet)
except QueryExpired:
pass
def import_documents(request):
def is_reserved_directory(doc):
return doc['fields']['type'] == 'directory' and doc['fields'][
'name'] in (Document2.HOME_DIR, Document2.TRASH_DIR)
try:
if request.FILES.get('documents'):
documents = request.FILES['documents'].read()
else:
documents = json.loads(request.POST.get('documents'))
documents = json.loads(documents)
except ValueError as e:
raise PopupException(
_('Failed to import documents, the file does not contain valid JSON.'
))
# Validate documents
if not _is_import_valid(documents):
raise PopupException(
_('Failed to import documents, the file does not contain the expected JSON schema for Hue documents.'
))
docs = []
uuids_map = dict((doc['fields']['uuid'], None) for doc in documents
if not is_reserved_directory(doc))
for doc in documents:
# Filter docs to import, ignoring reserved directories (home and Trash) and history docs
if not is_reserved_directory(doc):
# Remove any deprecated fields
if 'tags' in doc['fields']:
doc['fields'].pop('tags')
# If doc is not owned by current user, make a copy of the document with current user as owner
if doc['fields']['owner'][0] != request.user.username:
doc = _copy_document_with_owner(doc, request.user, uuids_map)
else: # Update existing doc or create new
doc = _create_or_update_document_with_owner(
doc, request.user, uuids_map)
# For oozie docs replace dependent uuids with the newly created ones
if doc['fields']['type'].startswith('oozie-'):
doc = _update_imported_oozie_document(doc, uuids_map)
# If the doc contains any history dependencies, ignore them
# NOTE: this assumes that each dependency is exported as an array using the natural PK [uuid, version, is_history]
deps_minus_history = [
dep for dep in doc['fields'].get('dependencies', [])
if len(dep) >= 3 and not dep[2]
]
doc['fields']['dependencies'] = deps_minus_history
# Replace illegal characters
if '/' in doc['fields']['name']:
new_name = doc['fields']['name'].replace('/', '-')
LOG.warn(
"Found illegal slash in document named: %s, renaming to: %s."
% (doc['fields']['name'], new_name))
doc['fields']['name'] = new_name
# Set last modified date to now
doc['fields']['last_modified'] = datetime.now().replace(
microsecond=0).isoformat()
docs.append(doc)
f = tempfile.NamedTemporaryFile(mode='w+', suffix='.json')
f.write(json.dumps(docs))
f.flush()
stdout = string_io()
try:
with transaction.atomic(
): # We wrap both commands to commit loaddata & sync
management.call_command(
'loaddata',
f.name,
verbosity=3,
traceback=True,
stdout=stdout,
commit=False
) # We need to use commit=False because commit=True will close the connection and make Document.objects.sync fail.
Document.objects.sync()
if request.POST.get('redirect'):
return redirect(request.POST.get('redirect'))
else:
return JsonResponse({
'status':
0,
'message':
stdout.getvalue(),
'count':
len(documents),
'created_count':
len([doc for doc in documents if doc['pk'] is None]),
'updated_count':
len([doc for doc in documents if doc['pk'] is not None]),
'username':
request.user.username,
'documents': [
dict([('name', doc['fields']['name']),
('uuid', doc['fields']['uuid']),
('type', doc['fields']['type']),
('owner', doc['fields']['owner'][0])])
for doc in docs
]
})
except Exception as e:
LOG.error('Failed to run loaddata command in import_documents:\n %s' %
stdout.getvalue())
return JsonResponse({'status': -1, 'message': smart_str(e)})
finally:
stdout.close()
db_form = DbForm(initial={'database': database}, databases=databases)
search_filter = request.GET.get('filter', '')
tables = db.get_tables_meta(database=database, table_names=search_filter) # SparkSql returns []
table_names = [table['name'] for table in tables]
except Exception, e:
raise PopupException(_('Failed to retrieve tables for database: %s' % database), detail=e)
database_meta = db.get_database(database)
if request.REQUEST.get("format", "html") == "json":
resp = JsonResponse({
'status': 0,
'database_meta': database_meta,
'tables': tables,
'table_names': table_names,
'search_filter': search_filter
})
else:
resp = render("tables.mako", request, {
'breadcrumbs': [
{
'name': database,
'url': reverse('metastore:show_tables', kwargs={'database': database})
}
],
'database_meta': database_meta,
'tables': tables,
'db_form': db_form,
'search_filter': search_filter,
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
scheduled_uuid = coordinator.data['properties'][
'workflow'] or coordinator.data['properties']['document']
if scheduled_uuid:
try:
document = Document2.objects.get(uuid=scheduled_uuid)
except Document2.DoesNotExist as e:
document = None
coordinator.data['properties']['workflow'] = ''
LOG.warn("Workflow with uuid %s doesn't exist: %s" %
(scheduled_uuid, e))
if document and document.is_trashed:
raise PopupException(
_('Your workflow %s has been trashed!') %
(document.name if document.name else ''))
if document and not document.can_read(request.user):
raise PopupException(
_('You don\'t have access to the workflow or document of this coordinator.'
))
else:
workflows = [
dict([('uuid', d.content_object.uuid),
('name', d.content_object.name)])
for d in Document.objects.available_docs(
Document2, request.user).filter(extra='workflow2')
]
if coordinator_id and not [
a for a in workflows
if a['uuid'] == coordinator.data['properties']['workflow']
]:
raise PopupException(
_('You don\'t have access to the workflow of this coordinator.'
))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [
dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(
request.user, include_managed=False).search_documents(
types=['oozie-workflow2'])
]
can_edit = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator':
coordinator.get_data_for_json(),
'credentials':
list(credentials.credentials.keys()),
'workflows':
workflows,
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit':
can_edit,
'layout':
django_mako.render_to_string(
'editor2/common_scheduler.mako',
{'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render(
'editor2/coordinator_editor.mako', request, {
'coordinator_json':
coordinator.to_json_for_html(),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflows_json':
json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_json':
json.dumps(can_edit)
})
