def notebooks(request):
editor_type = request.GET.get("type", "notebook")
if editor_type != "notebook":
if USE_NEW_EDITOR.get():
notebooks = [
doc.to_dict()
for doc in Document2.objects.documents(user=request.user).search_documents(
types=["query-%s" % editor_type]
)
]
else:
notebooks = [
d.content_object.to_dict()
for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra__startswith="query"))
if not d.content_object.is_history and d.content_object.type == "query-" + editor_type
]
else:
if USE_NEW_EDITOR.get():
notebooks = [
doc.to_dict()
for doc in Document2.objects.documents(user=request.user).search_documents(types=["notebook"])
]
else:
notebooks = [
d.content_object.to_dict()
for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra="notebook"))
if not d.content_object.is_history
]
return render(
"notebooks.mako",
request,
{"notebooks_json": json.dumps(notebooks, cls=JSONEncoderForHTML), "editor_type": editor_type},
)
def notebooks(request):
editor_type = request.GET.get('type', 'notebook')
if editor_type != 'notebook':
if USE_NEW_EDITOR.get():
notebooks = [doc.to_dict() for doc in Document2.objects.documents(
user=request.user).search_documents(types=['query-%s' % editor_type])]
else:
notebooks = [
d.content_object.to_dict()
for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra__startswith='query'))
if not d.content_object.is_history and d.content_object.type == 'query-' + editor_type
]
else:
if USE_NEW_EDITOR.get():
notebooks = [doc.to_dict() for doc in Document2.objects.documents(user=request.user).search_documents(types=['notebook'])]
else:
notebooks = [
d.content_object.to_dict()
for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra='notebook'))
if not d.content_object.is_history
]
return render('notebooks.mako', request, {
'notebooks_json': json.dumps(notebooks, cls=JSONEncoderForHTML),
'editor_type': editor_type
})
def edit_bundle(request):
bundle_id = request.GET.get('bundle')
doc = None
if bundle_id:
doc = Document2.objects.get(id=bundle_id)
bundle = Bundle(document=doc)
else:
bundle = Bundle()
bundle.set_workspace(request.user)
if USE_NEW_EDITOR.get():
coordinators = [dict([('id', d.id), ('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(request.user).search_documents(types=['oozie-coordinator2'])]
else:
coordinators = [dict([('id', d.content_object.id), ('uuid', d.content_object.uuid), ('name', d.content_object.name)])
for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
can_edit_json = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user))
return render('editor2/bundle_editor.mako', request, {
'bundle_json': bundle.to_json_for_html(),
'coordinators_json': json.dumps(coordinators, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'is_embeddable': request.GET.get('is_embeddable', False),
'can_edit_json': json.dumps(can_edit_json)
})
def edit_bundle(request):
bundle_id = request.GET.get('bundle')
doc = None
if bundle_id:
doc = Document2.objects.get(id=bundle_id)
bundle = Bundle(document=doc)
else:
bundle = Bundle()
bundle.set_workspace(request.user)
if USE_NEW_EDITOR.get():
coordinators = [dict([('id', d.id), ('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(request.user).search_documents(types=['oozie-coordinator2'])]
else:
coordinators = [dict([('id', d.content_object.id), ('uuid', d.content_object.uuid), ('name', d.content_object.name)])
for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
can_edit_json = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user))
return render('editor2/bundle_editor.mako', request, {
'bundle_json': bundle.to_json_for_html(),
'coordinators_json': json.dumps(coordinators, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'is_embeddable': request.GET.get('is_embeddable', False),
'can_edit_json': json.dumps(can_edit_json)
})
def index(request, is_mobile=False):
engine = request.GET.get('engine', 'solr')
cluster = request.POST.get('cluster', '""')
collection_id = request.GET.get('collection')
collections = get_engine(
request.user, engine,
cluster=cluster).datasets() if engine != 'report' else ['default']
if not collections:
if engine == 'solr':
return no_collections(request)
else:
return importer(request)
try:
collection_doc = Document2.objects.get(id=collection_id)
if USE_NEW_EDITOR.get():
collection_doc.can_read_or_exception(request.user)
else:
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception, e:
raise PopupException(
e,
title=
_("Dashboard does not exist or you don't have the permission to access it."
))
def handle_noargs(self, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(fs.DEFAULT_USER, fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
sample_user = install_sample_user()
management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2)
Document.objects.sync()
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
try:
# Don't overwrite
doc = Document.objects.get(object_id=1100713)
doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document.DoesNotExist:
LOG.warn('Sample pig script document not found.')
except Document2.DoesNotExist:
if doc.content_object:
data = doc.content_object.dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=sample_user,
parent_directory=examples_dir,
name=doc.name,
type='link-pigscript',
description=doc.description,
data=data)
LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def get_shared_search_collections(self):
# Those are the ones appearing in the menu
if USE_NEW_EDITOR.get():
return Document2.objects.filter(Q(owner=self.user) | Q(owner__username__in=SAMPLE_USER_OWNERS), type='search-dashboard').order_by('-id')
else:
docs = Document.objects.filter(Q(owner=self.user) | Q(owner__username__in=SAMPLE_USER_OWNERS), extra='search-dashboard')
return [d.content_object for d in docs.order_by('-id')]
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
can_edit_json = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
return render(
'editor2/workflow_editor.mako', request, {
'layout_json':
json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML),
'workflow_json':
json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflow_properties_json':
json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'subworkflows_json':
json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json':
json.dumps(can_edit_json),
'is_embeddable':
request.GET.get('is_embeddable', False),
})
def delete_job(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
if job.get('uuid'):
doc2 = Document2.objects.get(id=job['id'])
if USE_NEW_EDITOR.get():
doc2 = Document2.objects.get(id=job['id'])
doc2.can_write_or_exception(request.user)
doc2.trash()
else:
doc = doc2.doc.get()
doc.can_write_or_exception(request.user)
doc.delete()
doc2.delete()
else: # Old version
job = Job.objects.can_read_or_exception(request, job['object_id'])
Job.objects.can_edit_or_exception(request, job)
OldWorklow.objects.destroy(job, request.fs)
response = {}
request.info(_('Document deleted.') if len(jobs) > 1 else _('Document deleted.'))
return JsonResponse(response)
def delete_job(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
if job.get('uuid'):
doc2 = Document2.objects.get(id=job['id'])
if USE_NEW_EDITOR.get():
doc2 = Document2.objects.get(id=job['id'])
doc2.can_write_or_exception(request.user)
doc2.trash()
else:
doc = doc2.doc.get()
doc.can_write_or_exception(request.user)
doc.delete()
doc2.delete()
else: # Old version
job = Job.objects.can_read_or_exception(request, job['object_id'])
Job.objects.can_edit_or_exception(request, job)
OldWorklow.objects.destroy(job, request.fs)
response = {}
request.info(
_('Document deleted.') if len(jobs) > 1 else _('Document deleted.'))
return JsonResponse(response)
def index(request):
if request.user.is_superuser and request.COOKIES.get('hueLandingPage') != 'home':
return redirect(reverse('about:index'))
else:
if USE_NEW_EDITOR.get():
return home2(request)
else:
return home(request)
def index(request):
if request.user.is_superuser and request.COOKIES.get('hueLandingPage') != 'home':
return redirect(reverse('about:index'))
else:
if USE_NEW_EDITOR.get():
return home2(request)
else:
return home(request)
def install(self, django_user, interpreter=None):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
examples_dir = _get_example_directory(django_user)
document_type = self._document_type(self.type, interpreter)
notebook = import_saved_beeswax_query(query, interpreter=interpreter)
try:
# Could move PK from a uuid in queries.json at some point to handle name changes without duplicating.
# And move to a simpler json format at some point.
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
if doc2.parent_directory != examples_dir: # Recover from Trash or if moved
doc2.parent_directory = examples_dir
data = json.loads(doc2.data)
data['uuid'] = doc2.uuid
doc2.data = json.dumps(data) # Refresh content
doc2.save()
except Document2.DoesNotExist:
# Create new example
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid, # Must the same as in the notebook data
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=document_type,
description=self.desc,
data=data
)
# TODO: FK to connector object
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % doc2)
def notebooks(request):
editor_type = request.GET.get('type', 'notebook')
if editor_type != 'notebook':
if USE_NEW_EDITOR.get():
notebooks = [doc.to_dict() for doc in Document2.objects.documents(user=request.user).search_documents(types=['query-%s' % editor_type])]
else:
notebooks = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra__startswith='query')) if not d.content_object.is_history and d.content_object.type == 'query-' + editor_type]
else:
if USE_NEW_EDITOR.get():
notebooks = [doc.to_dict() for doc in Document2.objects.documents(user=request.user).search_documents(types=['notebook'])]
else:
notebooks = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, qfilter=Q(extra='notebook')) if not d.content_object.is_history]
return render('notebooks.mako', request, {
'notebooks_json': json.dumps(notebooks, cls=JSONEncoderForHTML),
'editor_type': editor_type
})
def handle(self, *args, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
sample_user = install_sample_user()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(
_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') %
{
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(
_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir,
'remote_data_dir': remote_data_dir
})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir,
remote_data_dir)
# Initialize doc2, whether editor script or link
doc2 = None
# Install editor pig script without doc1 link
LOG.info("Using Hue 4, will install pig editor sample.")
doc2 = self.install_pig_script(sample_user)
if USE_NEW_EDITOR.get():
# Get or create sample user directories
LOG.info("Creating sample user directories.")
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
# If document exists but has been trashed, recover from Trash
if doc2 and doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
# Share with default group
examples_dir.share(sample_user,
Document2Permission.READ_PERM,
groups=[get_default_user_group()])
def index(request):
if request.user.is_superuser and request.COOKIES.get('hueLandingPage') != 'home' and not IS_HUE_4.get():
return redirect(reverse('about:index'))
else:
if IS_HUE_4.get():
return redirect('desktop.views.hue')
elif USE_NEW_EDITOR.get():
return home2(request)
else:
return home(request)
def get_search_collections(self):
if USE_NEW_EDITOR.get():
return Document2.objects.documents(user=self.user).search_documents(
types=["search-dashboard"], order_by="-id"
)
else:
return [
d.content_object
for d in Document.objects.get_docs(self.user, Document2, extra="search-dashboard").order_by("-id")
]
def install(self, django_user, interpreter=None):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
document_type = self._document_type(self.type, interpreter)
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=document_type, is_history=False)
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query, interpreter=interpreter)
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid,
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=document_type,
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % (self.name,))
def get_shared_search_collections(self):
# Those are the ones appearing in the menu
if USE_NEW_EDITOR.get():
return Document2.objects.filter(
Q(owner=self.user) | Q(owner__username__in=SAMPLE_USER_OWNERS), type="search-dashboard"
).order_by("-id")
else:
docs = Document.objects.filter(
Q(owner=self.user) | Q(owner__username__in=SAMPLE_USER_OWNERS), extra="search-dashboard"
)
return [d.content_object for d in docs.order_by("-id")]
def install(self, django_user):
"""
Install queries. Raise InstallException on failure.
"""
LOG.info('Installing sample query: %s' % (self.name,))
try:
# Don't overwrite
query = SavedQuery.objects.get(owner=django_user, name=self.name, type=self.type)
except SavedQuery.DoesNotExist:
query = SavedQuery(owner=django_user, name=self.name, type=self.type, desc=self.desc)
# The data field needs to be a string. The sample file writes it
# as json (without encoding into a string) for readability.
query.data = json.dumps(self.data)
query.save()
LOG.info('Successfully installed sample design: %s' % (self.name,))
if USE_NEW_EDITOR.get():
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(django_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=django_user,
name=Document2.EXAMPLES_DIR
)
try:
# Don't overwrite
doc2 = Document2.objects.get(owner=django_user, name=self.name, type=self._document_type(self.type))
# If document exists but has been trashed, recover from Trash
if doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
except Document2.DoesNotExist:
# Create document from saved query
notebook = import_saved_beeswax_query(query)
data = notebook.get_data()
data['isSaved'] = True
uuid = data.get('uuid')
data = json.dumps(data)
doc2 = Document2.objects.create(
uuid=uuid,
owner=django_user,
parent_directory=examples_dir,
name=self.name,
type=self._document_type(self.type),
description=self.desc,
data=data
)
# Share with default group
examples_dir.share(django_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
LOG.info('Successfully installed sample query: %s' % (self.name,))
def get_search_collections(self):
if USE_NEW_EDITOR.get():
return Document2.objects.documents(
user=self.user).search_documents(types=['search-dashboard'],
order_by='-id')
else:
return [
d.content_object for d in Document.objects.get_docs(
self.user, Document2, extra='search-dashboard').order_by(
'-id')
]
def get_owner_search_collections(self):
if USE_NEW_EDITOR.get():
if self.user.is_superuser:
docs = Document2.objects.filter(type="search-dashboard")
else:
docs = Document2.objects.filter(type="search-dashboard", owner=self.user)
return docs
else:
if self.user.is_superuser:
docs = Document.objects.filter(extra="search-dashboard")
else:
docs = Document.objects.filter(extra="search-dashboard", owner=self.user)
return [d.content_object for d in docs.order_by("-id")]
def get_owner_search_collections(self):
if USE_NEW_EDITOR.get():
if self.user.is_superuser:
docs = Document2.objects.filter(type='search-dashboard')
else:
docs = Document2.objects.filter(type='search-dashboard', owner=self.user)
return docs
else:
if self.user.is_superuser:
docs = Document.objects.filter(extra='search-dashboard')
else:
docs = Document.objects.filter(extra='search-dashboard', owner=self.user)
return [d.content_object for d in docs.order_by('-id')]
def setUp(self):
self.client = make_logged_in_client(username="******", groupname="default", recreate=True, is_superuser=False)
self.client_not_me = make_logged_in_client(username="******", groupname="default", recreate=True, is_superuser=False)
self.user = User.objects.get(username="******")
self.user_not_me = User.objects.get(username="******")
self.old_home_path = '/home2' if USE_NEW_EDITOR.get() else '/home'
grant_access(self.user.username, self.user.username, "desktop")
grant_access(self.user_not_me.username, self.user_not_me.username, "desktop")
PigScript.objects.filter(owner=self.user).delete()
Document.objects.filter(owner=self.user).delete()
def list_editor_coordinators(request):
if USE_NEW_EDITOR.get():
docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-coordinator2'])
coordinators = [doc.to_dict() for doc in docs]
else:
coordinators = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
coordinators_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldCoordinator, request.user)]
if coordinators_v1:
coordinators.extend(coordinators_v1)
return render('editor2/list_editor_coordinators.mako', request, {
'coordinators_json': json.dumps(coordinators, cls=JSONEncoderForHTML),
})
def list_editor_bundles(request):
if USE_NEW_EDITOR.get():
docs = Document2.objects.documents(request.user).search_documents(types=['oozie-bundle2'])
bundles = [doc.to_dict() for doc in docs]
else:
bundles = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='bundle2')]
bundles_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldBundle, request.user)]
if bundles_v1:
bundles.extend(bundles_v1)
return render('editor2/list_editor_bundles.mako', request, {
'bundles_json': json.dumps(bundles, cls=JSONEncoderForHTML)
})
def list_editor_workflows(request):
if USE_NEW_EDITOR.get():
docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-workflow2'])
workflows = [doc.to_dict() for doc in docs]
else:
workflows = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='workflow2')]
workflows_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldWorklow, request.user) if job.managed]
if workflows_v1:
workflows.extend(workflows_v1)
return render('editor2/list_editor_workflows.mako', request, {
'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML),
})
def list_editor_bundles(request):
if USE_NEW_EDITOR.get():
docs = Document2.objects.documents(request.user).search_documents(types=['oozie-bundle2'])
bundles = [doc.to_dict() for doc in docs]
else:
bundles = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='bundle2')]
bundles_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldBundle, request.user)]
if bundles_v1:
bundles.extend(bundles_v1)
return render('editor2/list_editor_bundles.mako', request, {
'bundles_json': json.dumps(bundles, cls=JSONEncoderForHTML),
})
def list_editor_coordinators(request):
if USE_NEW_EDITOR.get():
docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-coordinator2'])
coordinators = [doc.to_dict() for doc in docs]
else:
coordinators = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='coordinator2')]
coordinators_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldCoordinator, request.user)]
if coordinators_v1:
coordinators.extend(coordinators_v1)
return render('editor2/list_editor_coordinators.mako', request, {
'coordinators_json': json.dumps(coordinators, cls=JSONEncoderForHTML)
})
def list_editor_workflows(request):
if USE_NEW_EDITOR.get():
docs = Document2.objects.documents(user=request.user).search_documents(types=['oozie-workflow2'])
workflows = [doc.to_dict() for doc in docs]
else:
workflows = [d.content_object.to_dict() for d in Document.objects.get_docs(request.user, Document2, extra='workflow2')]
workflows_v1 = [job.doc.get().to_dict() for job in Document.objects.available(OldWorklow, request.user) if job.managed]
if workflows_v1:
workflows.extend(workflows_v1)
return render('editor2/list_editor_workflows.mako', request, {
'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML)
})
def index(request):
try:
user_hue_version = json.loads(UserPreferences.objects.get(user=request.user, key='hue_version').value)
IS_HUE_4.set_for_testing(user_hue_version >= 4)
except UserPreferences.DoesNotExist:
pass
if request.user.is_superuser and request.COOKIES.get('hueLandingPage') != 'home' and not IS_HUE_4.get():
return redirect(reverse('about:index'))
else:
if IS_HUE_4.get():
return redirect('desktop.views.hue')
elif USE_NEW_EDITOR.get():
return home2(request)
else:
return home(request)
def index(request, is_mobile=False, is_embeddable=False):
hue_collections = SearchController(request.user).get_search_collections()
collection_id = request.GET.get("collection")
if not hue_collections or not collection_id:
return admin_collections(request, True, is_mobile)
try:
collection_doc = Document2.objects.get(id=collection_id)
if USE_NEW_EDITOR.get():
collection_doc.can_read_or_exception(request.user)
else:
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception, e:
raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
def index(request, is_mobile=False, is_embeddable=False):
hue_collections = DashboardController(request.user).get_search_collections()
collection_id = request.GET.get('collection')
if not hue_collections or not collection_id:
return admin_collections(request, True, is_mobile)
try:
collection_doc = Document2.objects.get(id=collection_id)
if USE_NEW_EDITOR.get():
collection_doc.can_read_or_exception(request.user)
else:
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception, e:
raise PopupException(e, title=_("Dashboard does not exist or you don't have the permission to access it."))
def decorate(request, *args, **kwargs):
collection_json = json.loads(request.POST.get('collection', '{}'))
if collection_json['id']:
try:
doc2 = Document2.objects.get(id=collection_json['id'])
if USE_NEW_EDITOR.get():
doc2.can_write_or_exception(request.user)
else:
doc2.doc.get().can_write_or_exception(request.user)
except Document2.DoesNotExist:
message = _("Dashboard does not exist or you don't have the permission to access it.")
raise PopupException(message)
return view_func(request, *args, **kwargs)
def handle(self, *args, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
sample_user = install_sample_user()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Initialize doc2, whether editor script or link
doc2 = None
# Install editor pig script without doc1 link
LOG.info("Using Hue 4, will install pig editor sample.")
doc2 = self.install_pig_script(sample_user)
if USE_NEW_EDITOR.get():
# Get or create sample user directories
LOG.info("Creating sample user directories.")
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
# If document exists but has been trashed, recover from Trash
if doc2 and doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def _get_workflows(user):
if USE_NEW_EDITOR.get():
workflows = [{
'name': workflow.name,
'owner': workflow.owner.username,
'value': workflow.uuid,
'id': workflow.id
} for workflow in [doc for doc in Document2.objects.documents(user).search_documents(types=['oozie-workflow2']).order_by('-id')]
]
else:
workflows = [{
'name': workflow.name,
'owner': workflow.owner.username,
'value': workflow.uuid,
'id': workflow.id
} for workflow in [d.content_object for d in Document.objects.get_docs(user, Document2, extra='workflow2').order_by('-id')]
]
return workflows
def _get_workflows(user):
if USE_NEW_EDITOR.get():
workflows = [{
'name': workflow.name,
'owner': workflow.owner.username,
'value': workflow.uuid,
'id': workflow.id
} for workflow in [doc for doc in Document2.objects.documents(user).search_documents(types=['oozie-workflow2']).order_by('-id')]
]
else:
workflows = [{
'name': workflow.name,
'owner': workflow.owner.username,
'value': workflow.uuid,
'id': workflow.id
} for workflow in [d.content_object for d in Document.objects.get_docs(user, Document2, extra='workflow2').order_by('-id')]
]
return workflows
def index(request):
is_hue_4 = IS_HUE_4.get() or DISABLE_HUE_3.get()
if is_hue_4:
try:
user_hue_version = json.loads(UserPreferences.objects.get(user=request.user, key='hue_version').value)
is_hue_4 = user_hue_version >= 4 or DISABLE_HUE_3.get()
except UserPreferences.DoesNotExist:
pass
if request.user.is_superuser and request.COOKIES.get('hueLandingPage') != 'home' and not IS_HUE_4.get():
return redirect(reverse('about:index'))
else:
if is_hue_4:
return redirect('desktop_views_hue')
elif USE_NEW_EDITOR.get():
return redirect('desktop_views_home2')
else:
return home(request)
def decorate(request, *args, **kwargs):
collection_json = json.loads(request.POST.get('collection', '{}'))
if collection_json['id']:
try:
doc2 = Document2.objects.get(id=collection_json['id'])
if USE_NEW_EDITOR.get():
doc2.can_write_or_exception(request.user)
else:
doc2.doc.get().can_write_or_exception(request.user)
except Document2.DoesNotExist:
message = _(
"Dashboard does not exist or you don't have the permission to access it."
)
raise PopupException(message)
return view_func(request, *args, **kwargs)
def decorate(request, *args, **kwargs):
doc_id = uuid = doc2 = None
try:
if request.GET.get('workflow'):
workflow_id = request.GET.get('workflow')
if workflow_id.isdigit():
doc_id = workflow_id
else:
uuid = workflow_id
elif request.GET.get('uuid'):
uuid = request.GET.get('uuid')
elif request.GET.get('coordinator'):
doc_id = request.GET.get('coordinator')
elif request.GET.get('bundle'):
doc_id = request.GET.get('bundle')
elif 'doc_id' in kwargs:
doc_id = kwargs['doc_id']
if doc_id and not doc_id.isdigit():
uuid = doc_id
doc_id = None
if doc_id is not None:
doc2 = Document2.objects.get(id=doc_id)
elif uuid is not None:
# TODO: The commented line should be used once we fully transition to doc2
# doc2 = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type=None)
doc2 = Document2.objects.filter(
uuid=uuid).order_by('-last_modified').first()
if doc2:
if USE_NEW_EDITOR.get():
doc2.can_read_or_exception(request.user)
else:
doc2.doc.get().can_read_or_exception(request.user)
except Document2.DoesNotExist:
raise PopupException(
_('Job with %(key)s=%(value)s does not exist') % {
'key': 'id' if doc_id else 'uuid',
'value': doc_id or uuid
})
return view_func(request, *args, **kwargs)
def setUp(self):
self.client = make_logged_in_client(username="******",
groupname="default",
recreate=True,
is_superuser=False)
self.client_not_me = make_logged_in_client(username="******",
groupname="default",
recreate=True,
is_superuser=False)
self.user = User.objects.get(username="******")
self.user_not_me = User.objects.get(username="******")
self.old_home_path = '/home2' if USE_NEW_EDITOR.get() else '/home'
grant_access(self.user.username, self.user.username, "desktop")
grant_access(self.user_not_me.username, self.user_not_me.username,
"desktop")
PigScript.objects.filter(owner=self.user).delete()
Document.objects.filter(owner=self.user).delete()
def decorate(request, *args, **kwargs):
doc_id = uuid = doc2 = None
try:
if request.REQUEST.get('workflow'):
workflow_id = request.REQUEST.get('workflow')
if workflow_id.isdigit():
doc_id = workflow_id
else:
uuid = workflow_id
elif request.GET.get('uuid'):
uuid = request.GET.get('uuid')
elif request.GET.get('coordinator'):
doc_id = request.GET.get('coordinator')
elif request.GET.get('bundle'):
doc_id = request.GET.get('bundle')
elif 'doc_id' in kwargs:
doc_id = kwargs['doc_id']
if doc_id and not doc_id.isdigit():
uuid = doc_id
doc_id = None
if doc_id is not None:
doc2 = Document2.objects.get(id=doc_id)
elif uuid is not None:
# TODO: The commented line should be used once we fully transition to doc2
# doc2 = Document2.objects.get_by_uuid(user=request.user, uuid=uuid, perm_type=None)
doc2 = Document2.objects.filter(uuid=uuid).order_by('-last_modified').first()
if doc2:
if USE_NEW_EDITOR.get():
doc2.can_read_or_exception(request.user)
else:
doc2.doc.get().can_read_or_exception(request.user)
except Document2.DoesNotExist:
raise PopupException(_('Job with %(key)s=%(value)s does not exist') %
{'key': 'id' if doc_id else 'uuid', 'value': doc_id or uuid})
return view_func(request, *args, **kwargs)
def decorate(request, *args, **kwargs):
doc_id = None
job = json.loads(request.POST.get('workflow', '{}'))
if not job:
job = json.loads(request.POST.get('coordinator', '{}'))
elif not job:
job = json.loads(request.POST.get('bundle', '{}'))
if job and job.get('id'):
doc_id = job.get('id')
try:
doc2 = Document2.objects.get(id=job['id'])
if USE_NEW_EDITOR.get():
doc2.can_write_or_exception(request.user)
else:
doc2.doc.get().can_write_or_exception(request.user)
except Document.DoesNotExist:
raise PopupException(_('Job %(id)s does not exist') % {'id': doc_id})
return view_func(request, *args, **kwargs)
def decorate(request, *args, **kwargs):
doc_id = None
job = json.loads(request.POST.get('workflow', '{}'))
if not job:
job = json.loads(request.POST.get('coordinator', '{}'))
elif not job:
job = json.loads(request.POST.get('bundle', '{}'))
if job and job.get('id'):
doc_id = job.get('id')
try:
doc2 = Document2.objects.get(id=job['id'])
if USE_NEW_EDITOR.get():
doc2.can_write_or_exception(request.user)
else:
doc2.doc.get().can_write_or_exception(request.user)
except Document.DoesNotExist:
raise PopupException(_('Job %(id)s does not exist') % {'id': doc_id})
return view_func(request, *args, **kwargs)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
scheduled_uuid = coordinator.data['properties'][
'workflow'] or coordinator.data['properties']['document']
if scheduled_uuid:
document = Document2.objects.get(uuid=scheduled_uuid)
if not document.can_read(request.user):
raise PopupException(
_('You don\'t have access to the workflow or document of this coordinator.'
))
else:
workflows = [
dict([('uuid', d.content_object.uuid),
('name', d.content_object.name)])
for d in Document.objects.available_docs(
Document2, request.user).filter(extra='workflow2')
]
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
if IS_HUE_4.get():
# Install editor oozie examples without doc1 link
LOG.info("Using Hue 4, will install oozie editor samples.")
example_jobs = []
example_jobs.append(self._install_mapreduce_example())
example_jobs.append(self._install_java_example())
example_jobs.append(self._install_spark_example())
example_jobs.append(self._install_pyspark_example())
# If documents exist but have been trashed, recover from Trash
for doc in example_jobs:
if doc is not None and doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
elif USE_NEW_EDITOR.get():
# Install as link-workflow doc2 to old Job Designs
docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
# Don't overwrite
doc2, created = Document2.objects.get_or_create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data
)
LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
if not IS_HUE_4.get():
self.install_examples()
Document.objects.sync()
coordinator = Coordinator()
coordinator.set_workspace(request.user)
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
workflows = [dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(request.user).search_documents(types=['oozie-workflow2'])]
else:
workflows = [dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)])
for d in Document.objects.available_docs(Document2, request.user).filter(extra='workflow2')]
if coordinator_id and not filter(lambda a: a['uuid'] == coordinator.data['properties']['workflow'], workflows):
raise PopupException(_('You don\'t have access to the workflow of this coordinator.'))
return render('editor2/coordinator_editor.mako', request, {
'coordinator_json': coordinator.to_json_for_html(),
'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML),
'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
scheduled_uuid = coordinator.data['properties'][
'workflow'] or coordinator.data['properties']['document']
if scheduled_uuid:
try:
document = Document2.objects.get(uuid=scheduled_uuid)
except Document2.DoesNotExist as e:
document = None
coordinator.data['properties']['workflow'] = ''
LOG.warning("Workflow with uuid %s doesn't exist: %s" %
(scheduled_uuid, e))
if document and document.is_trashed:
raise PopupException(
_('Your workflow %s has been trashed!') %
(document.name if document.name else ''))
if document and not document.can_read(request.user):
raise PopupException(
_('You don\'t have access to the workflow or document of this coordinator.'
))
else:
workflows = [
dict([('uuid', d.content_object.uuid),
('name', d.content_object.name)])
for d in Document.objects.available_docs(
Document2, request.user).filter(extra='workflow2')
]
if coordinator_id and not [
a for a in workflows
if a['uuid'] == coordinator.data['properties']['workflow']
]:
raise PopupException(
_('You don\'t have access to the workflow of this coordinator.'
))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [
dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(
request.user, include_managed=False).search_documents(
types=['oozie-workflow2'])
]
can_edit = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator':
coordinator.get_data_for_json(),
'credentials':
list(credentials.credentials.keys()),
'workflows':
workflows,
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit':
can_edit,
'layout':
django_mako.render_to_string(
'editor2/common_scheduler.mako',
{'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render(
'editor2/coordinator_editor.mako', request, {
'coordinator_json':
coordinator.to_json_for_html(),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflows_json':
json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_json':
json.dumps(can_edit)
})
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
workflows = [dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(request.user, include_managed=True).search_documents(types=['oozie-workflow2'])]
else:
workflows = [dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)])
for d in Document.objects.available_docs(Document2, request.user).filter(extra='workflow2')]
if coordinator_id and not filter(lambda a: a['uuid'] == coordinator.data['properties']['workflow'], workflows): # In Hue 4, use dependencies instead
raise PopupException(_('You don\'t have access to the workflow of this coordinator.'))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(request.user, include_managed=False).search_documents(types=['oozie-workflow2'])]
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
return _edit_workflow(request, doc, workflow)
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
can_edit_json = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user))
return render('editor2/workflow_editor.mako', request, {
'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML),
'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML),
'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML),
'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json': json.dumps(can_edit_json),
'is_embeddable': request.GET.get('is_embeddable', False),
})
@check_editor_access_permission
def new_workflow(request):
def handle(self, *args, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
with transaction.atomic():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False)
if IS_HUE_4.get():
# Install editor oozie examples without doc1 link
LOG.info("Using Hue 4, will install oozie editor samples.")
example_jobs = []
example_jobs.append(self._install_mapreduce_example())
example_jobs.append(self._install_java_example())
example_jobs.append(self._install_spark_example())
example_jobs.append(self._install_pyspark_example())
# If documents exist but have been trashed, recover from Trash
for doc in example_jobs:
if doc is not None and doc.parent_directory != examples_dir:
doc.parent_directory = examples_dir
doc.save()
elif USE_NEW_EDITOR.get():
# Install as link-workflow doc2 to old Job Designs
docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
# Don't overwrite
doc2, created = Document2.objects.get_or_create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data
)
LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
if not IS_HUE_4.get():
self.install_examples()
Document.objects.sync()
def index(request, is_mobile=False):
engine = request.GET.get('engine', 'solr')
cluster = request.POST.get('cluster', '""')
collection_id = request.GET.get('collection')
collections = get_engine(
request.user, engine,
cluster=cluster).datasets() if engine != 'report' else ['default']
if not collections:
if engine == 'solr':
return no_collections(request)
else:
return importer(request)
try:
collection_doc = Document2.objects.get(id=collection_id)
if USE_NEW_EDITOR.get():
collection_doc.can_read_or_exception(request.user)
else:
collection_doc.doc.get().can_read_or_exception(request.user)
collection = Collection2(request.user, document=collection_doc)
except Exception as e:
raise PopupException(
e,
title=
_("Dashboard does not exist or you don't have the permission to access it."
))
query = {'qs': [{'q': ''}], 'fqs': [], 'start': 0}
if request.method == 'GET':
if 'q' in request.GET:
query['qs'][0]['q'] = antixss(request.GET.get('q', ''))
if 'qd' in request.GET:
query['qd'] = antixss(request.GET.get('qd', ''))
template = 'search.mako'
if is_mobile:
template = 'search_m.mako'
engine = collection.data['collection'].get('engine', 'solr')
return render(
template, request, {
'collection':
collection,
'query':
json.dumps(query),
'initial':
json.dumps({
'collections': collections,
'layout': DEFAULT_LAYOUT,
'qb_layout': QUERY_BUILDER_LAYOUT,
'text_search_layout': TEXT_SEARCH_LAYOUT,
'is_latest': _get_latest(),
'engines': get_engines(request.user)
}),
'is_owner':
collection_doc.can_write(request.user) if USE_NEW_EDITOR.get() else
collection_doc.doc.get().can_write(request.user),
'can_edit_index':
can_edit_index(request.user),
'is_embeddable':
request.GET.get('is_embeddable', False),
'mobile':
is_mobile,
'is_report':
collection.data['collection'].get('engine') == 'report'
})
template = 'search.mako'
if is_mobile:
template = 'search_m.mako'
engine = collection.data['collection']['engine']
return render(template, request, {
'collection': collection,
'query': json.dumps(query),
'initial': json.dumps({
'collections': [],
'layout': DEFAULT_LAYOUT,
'qb_layout': QUERY_BUILDER_LAYOUT,
'text_search_layout': TEXT_SEARCH_LAYOUT,
'is_latest': _get_latest(),
'engines': get_engines(request.user)
}),
'is_owner': collection_doc.can_write(request.user) if USE_NEW_EDITOR.get() else collection_doc.doc.get().can_write(request.user),
'can_edit_index': can_edit_index(request.user),
'is_embeddable': request.GET.get('is_embeddable', False),
'mobile': is_mobile,
'is_report': collection.data['collection'].get('engine') == 'report'
})
def index_m(request):
return index(request, True)
def new_search(request):
engine = request.GET.get('engine', 'solr')
cluster = request.POST.get('cluster','""')
collections = get_engine(request.user, engine, cluster=cluster).datasets() if engine != 'report' else ['default']
if not collections:
if engine == 'solr':
def handle(self, *args, **options):
fs = cluster.get_hdfs()
create_directories(fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
sample_user = install_sample_user()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = paths.get_thirdparty_root("sample_data")
remote_data_dir = fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
fs.do_as_user(sample_user.username, fs.copyFromLocal, local_dir, remote_data_dir)
# Initialize doc2, whether editor script or link
doc2 = None
if IS_HUE_4.get():
# Install editor pig script without doc1 link
LOG.info("Using Hue 4, will install pig editor sample.")
doc2 = self.install_pig_script(sample_user)
else:
# Install old pig script fixture
LOG.info("Using Hue 3, will install pig script fixture.")
with transaction.atomic():
management.call_command('loaddata', 'initial_pig_examples.json', verbosity=2, commit=False)
Document.objects.sync()
if USE_NEW_EDITOR.get():
# Get or create sample user directories
LOG.info("Creating sample user directories.")
home_dir = Directory.objects.get_home_directory(sample_user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=sample_user,
name=Document2.EXAMPLES_DIR)
if not IS_HUE_4.get():
try:
# Don't overwrite
doc = Document.objects.get(object_id=1100713)
doc2 = Document2.objects.get(owner=sample_user, name=doc.name, type='link-pigscript')
except Document.DoesNotExist:
LOG.warn('Sample pig script document not found.')
except Document2.DoesNotExist:
if doc.content_object:
data = doc.content_object.dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=sample_user,
parent_directory=examples_dir,
name=doc.name,
type='link-pigscript',
description=doc.description,
data=data)
LOG.info('Successfully installed sample link to pig script: %s' % (doc2.name,))
# If document exists but has been trashed, recover from Trash
if doc2 and doc2.parent_directory != examples_dir:
doc2.parent_directory = examples_dir
doc2.save()
# Share with default group
examples_dir.share(sample_user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
# Some django-wide URLs
dynamic_patterns = [
url(r'^hue/accounts/login',
desktop_auth_views.dt_login,
name='desktop_auth_views_dt_login'),
url(r'^accounts/login/?$', desktop_auth_views.dt_login), # Deprecated
url(r'^accounts/logout/?$', desktop_auth_views.dt_logout,
{'next_page': '/'}),
url(r'^profile$', desktop_auth_views.profile),
url(r'^login/oauth/?$', desktop_auth_views.oauth_login),
url(r'^login/oauth_authenticated/?$',
desktop_auth_views.oauth_authenticated),
url(r'^hue/oidc_failed', desktop_auth_views.oidc_failed),
]
if USE_NEW_EDITOR.get():
dynamic_patterns += [
url(r'^home/?$', desktop_views.home2, name='desktop_views_home2'),
url(r'^home2$', desktop_views.home, name='desktop_views_home'),
url(r'^home_embeddable$', desktop_views.home_embeddable),
]
else:
dynamic_patterns += [
url(r'^home$', desktop_views.home, name='desktop_views_home'),
url(r'^home2$', desktop_views.home2, name='desktop_views_home2')
]
dynamic_patterns += [
url(r'^logs$', desktop_views.log_view, name="desktop.views.log_view"),
url(r'^desktop/log_analytics$', desktop_views.log_analytics),
url(r'^desktop/log_js_error$', desktop_views.log_js_error),
def handle_noargs(self, **options):
self.user = install_sample_user()
self.fs = cluster.get_hdfs()
LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get())
create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()])
remote_dir = REMOTE_SAMPLE_DIR.get()
# Copy examples binaries
for name in os.listdir(LOCAL_SAMPLE_DIR.get()):
local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name)
remote_data_dir = self.fs.join(remote_dir, name)
LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Copy sample data
local_dir = LOCAL_SAMPLE_DATA_DIR.get()
remote_data_dir = self.fs.join(remote_dir, 'data')
LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % {
'local_dir': local_dir, 'remote_data_dir': remote_data_dir})
self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir)
# Load jobs
LOG.info(_("Installing examples..."))
if ENABLE_V2.get():
management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2)
# Get or create sample user directories
home_dir = Directory.objects.get_home_directory(self.user)
examples_dir, created = Directory.objects.get_or_create(
parent_directory=home_dir,
owner=self.user,
name=Document2.EXAMPLES_DIR
)
if USE_NEW_EDITOR.get():
docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user)
for doc in docs:
if doc.content_object:
data = doc.content_object.data_dict
data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id})
data = json.dumps(data)
doc2 = Document2.objects.create(
owner=self.user,
parent_directory=examples_dir,
name=doc.name,
type='link-workflow',
description=doc.description,
data=data)
LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,))
# Share oozie examples with default group
oozie_examples = Document2.objects.filter(
type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'],
owner=self.user,
parent_directory=None
)
oozie_examples.update(parent_directory=examples_dir)
examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
self.install_examples()
Document.objects.sync()