def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) self.install_examples() Document.objects.sync()
def list_oozie_bundle(request, job_id): oozie_bundle = check_job_access_permission(request, job_id) # Cross reference the submission history (if any) bundle = None try: if ENABLE_V2.get(): bundle = get_history().get_bundle_from_config(oozie_bundle.conf_dict) else: bundle = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job bundle for job_id=%s", job_id) if request.GET.get('format') == 'json': return_obj = { 'id': oozie_bundle.id, 'status': oozie_bundle.status, 'progress': oozie_bundle.get_progress(), 'endTime': format_time(oozie_bundle.endTime), 'actions': massaged_bundle_actions_for_json(oozie_bundle) } return HttpResponse(json.dumps(return_obj).replace('\\\\', '\\'), content_type="application/json") return render('dashboard/list_oozie_bundle.mako', request, { 'oozie_bundle': oozie_bundle, 'bundle': bundle, 'has_job_edition_permission': has_job_edition_permission, })
def list_oozie_bundle(request, job_id): oozie_bundle = check_job_access_permission(request, job_id) # Cross reference the submission history (if any) bundle = None try: if ENABLE_V2.get(): bundle = get_history().get_bundle_from_config(oozie_bundle.conf_dict) else: bundle = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job bundle for job_id=%s", job_id) if request.GET.get('format') == 'json': return_obj = { 'id': oozie_bundle.id, 'user': oozie_bundle.user, 'name': oozie_bundle.bundleJobName, 'status': oozie_bundle.status, 'progress': oozie_bundle.get_progress(), 'endTime': format_time(oozie_bundle.endTime), 'actions': massaged_bundle_actions_for_json(oozie_bundle), 'submitted': format_time(oozie_bundle.kickoffTime), 'doc_url': bundle.get_absolute_url() if bundle else '', 'canEdit': has_job_edition_permission(oozie_bundle, request.user), } return HttpResponse(json.dumps(return_obj).replace('\\\\', '\\'), content_type="application/json") return render('dashboard/list_oozie_bundle.mako', request, { 'oozie_bundle': oozie_bundle, 'bundle': bundle, 'has_job_edition_permission': has_job_edition_permission, })
def list_oozie_bundle(request, job_id): oozie_bundle = check_job_access_permission(request, job_id) # Cross reference the submission history (if any) bundle = None try: if ENABLE_V2.get(): bundle = get_history().get_bundle_from_config(oozie_bundle.conf_dict) else: bundle = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job bundle for job_id=%s", job_id) if request.GET.get("format") == "json": return_obj = { "id": oozie_bundle.id, "status": oozie_bundle.status, "progress": oozie_bundle.get_progress(), "endTime": format_time(oozie_bundle.endTime), "actions": massaged_bundle_actions_for_json(oozie_bundle), } return HttpResponse(json.dumps(return_obj).replace("\\\\", "\\"), content_type="application/json") return render( "dashboard/list_oozie_bundle.mako", request, {"oozie_bundle": oozie_bundle, "bundle": bundle, "has_job_edition_permission": has_job_edition_permission}, )
def list_oozie_coordinator(request, job_id): kwargs = {'cnt': 50, 'filters': []} kwargs['offset'] = request.GET.get('offset', 1) if request.GET.getlist('status'): kwargs['filters'].extend([('status', status) for status in request.GET.getlist('status')]) oozie_coordinator = check_job_access_permission(request, job_id, **kwargs) # Cross reference the submission history (if any) coordinator = get_history().get_coordinator_from_config(oozie_coordinator.conf_dict) try: if not ENABLE_V2.get(): coordinator = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job coordinator for job_id=%s", job_id) oozie_bundle = None if request.GET.get('bundle_job_id'): try: oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) except: LOG.exception("Ignoring error getting oozie bundle for job_id=%s", job_id) if request.GET.get('format') == 'json': actions = massaged_coordinator_actions_for_json(oozie_coordinator, oozie_bundle) return_obj = { 'id': oozie_coordinator.id, 'status': oozie_coordinator.status, 'progress': oozie_coordinator.get_progress(), 'nextTime': format_time(oozie_coordinator.nextMaterializedTime), 'endTime': format_time(oozie_coordinator.endTime), 'actions': actions, 'total_actions': oozie_coordinator.total, 'doc_url': coordinator.get_absolute_url() if coordinator else '', } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_coordinator.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_coordinator.id, 'parent_id': oozie_coordinator.id } oozie_slas = oozie_api.get_oozie_slas(**params) enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get() update_coord_form = UpdateCoordinatorForm(oozie_coordinator=oozie_coordinator) return render('dashboard/list_oozie_coordinator.mako', request, { 'oozie_coordinator': oozie_coordinator, 'oozie_slas': oozie_slas, 'coordinator': coordinator, 'oozie_bundle': oozie_bundle, 'has_job_edition_permission': has_job_edition_permission, 'enable_cron_scheduling': enable_cron_scheduling, 'update_coord_form': update_coord_form, })
def list_oozie_coordinator(request, job_id): kwargs = {"cnt": 50, "filters": []} kwargs["offset"] = request.GET.get("offset", 1) if request.GET.getlist("status"): kwargs["filters"].extend([("status", status) for status in request.GET.getlist("status")]) oozie_coordinator = check_job_access_permission(request, job_id, **kwargs) # Cross reference the submission history (if any) coordinator = get_history().get_coordinator_from_config(oozie_coordinator.conf_dict) try: if not ENABLE_V2.get(): coordinator = get_history().objects.get(oozie_job_id=job_id).job.get_full_node() except: LOG.exception("Ignoring error getting oozie job coordinator for job_id=%s", job_id) oozie_bundle = None if request.GET.get("bundle_job_id"): try: oozie_bundle = check_job_access_permission(request, request.GET.get("bundle_job_id")) except: LOG.exception("Ignoring error getting oozie bundle for job_id=%s", job_id) if request.GET.get("format") == "json": actions = massaged_coordinator_actions_for_json(oozie_coordinator, oozie_bundle) return_obj = { "id": oozie_coordinator.id, "status": oozie_coordinator.status, "progress": oozie_coordinator.get_progress(), "nextTime": format_time(oozie_coordinator.nextMaterializedTime), "endTime": format_time(oozie_coordinator.endTime), "actions": actions, "total_actions": oozie_coordinator.total, } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_coordinator.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = {"id": oozie_coordinator.id, "parent_id": oozie_coordinator.id} oozie_slas = oozie_api.get_oozie_slas(**params) enable_cron_scheduling = ENABLE_CRON_SCHEDULING.get() update_coord_form = UpdateCoordinatorForm(oozie_coordinator=oozie_coordinator) return render( "dashboard/list_oozie_coordinator.mako", request, { "oozie_coordinator": oozie_coordinator, "oozie_slas": oozie_slas, "coordinator": coordinator, "oozie_bundle": oozie_bundle, "has_job_edition_permission": has_job_edition_permission, "enable_cron_scheduling": enable_cron_scheduling, "update_coord_form": update_coord_form, }, )
def test_submit_single_action(self): wf_doc = save_temp_workflow(MockOozieApi.JSON_WORKFLOW_LIST[5], self.user) reset = ENABLE_V2.set_for_testing(True) try: response = self.c.get(reverse('oozie:submit_single_action', args=[wf_doc.id, '3f107997-04cc-8733-60a9-a4bb62cebabc'])) assert_equal([{'name':'Dryrun', 'value': False}, {'name':'ls_arg', 'value': '-l'}], response.context['params_form'].initial) except Exception, ex: logging.exception(ex)
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = {} credentials = None doc = None hue_workflow = None hue_coord = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict) # When a workflow is submitted by a coordinator if not hue_workflow and hue_coord: hue_workflow = hue_coord.workflow if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() if not workflow_data.get('layout'): try: workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow) except Exception, e: LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)) workflow_graph = '' credentials = Credentials() except:
def import_workflow(request): if ENABLE_V2.get(): raise PopupException( '/oozie/import_workflow is deprecated in the version 2 of Editor') workflow = Workflow.objects.new_workflow(request.user) if request.method == 'POST': workflow_form = ImportWorkflowForm(request.POST, request.FILES, instance=workflow) if workflow_form.is_valid(): if workflow_form.cleaned_data.get('resource_archive'): # Upload resources to workspace source = workflow_form.cleaned_data.get('resource_archive') if source.name.endswith('.zip'): workflow.save() Workflow.objects.initialize(workflow, request.fs) temp_path = archive_factory(source).extract() request.fs.copyFromLocal(temp_path, workflow.deployment_dir) shutil.rmtree(temp_path) else: raise PopupException(_('Archive should be a Zip.')) else: workflow.save() Workflow.objects.initialize(workflow, request.fs) workflow.managed = True workflow.save() workflow_definition = workflow_form.cleaned_data[ 'definition_file'].read() try: _import_workflow(fs=request.fs, workflow=workflow, workflow_definition=workflow_definition) request.info(_('Workflow imported')) return redirect( reverse('oozie:edit_workflow', kwargs={'workflow': workflow.id})) except Exception as e: request.error(_('Could not import workflow: %s' % e)) Workflow.objects.destroy(workflow, request.fs) raise PopupException(_('Could not import workflow.'), detail=e) else: request.error(_('Errors on the form: %s') % workflow_form.errors) else: workflow_form = ImportWorkflowForm(instance=workflow) return render('editor/import_workflow.mako', request, { 'workflow_form': workflow_form, 'workflow': workflow, })
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = {} credentials = None doc = None hue_workflow = None hue_coord = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() if not workflow_data.get('layout'): try: workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow) except Exception, e: LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)) workflow_graph = '' credentials = Credentials() except:
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = {} credentials = None doc = None hue_workflow = None hue_coord = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict) # When a workflow is submitted by a coordinator if not hue_workflow and hue_coord and hue_coord.workflow.document: hue_workflow = hue_coord.workflow if hue_coord and hue_coord.workflow and hue_coord.workflow.document: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() except Exception, e: LOG.exception("Error generating full page for running workflow %s with exception: %s" % (job_id, e.message)) finally:
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def test_submit_single_action(self): wf_doc = save_temp_workflow(MockOozieApi.JSON_WORKFLOW_LIST[5], self.user) reset = ENABLE_V2.set_for_testing(True) try: response = self.c.get( reverse( 'oozie:submit_single_action', args=[wf_doc.id, '3f107997-04cc-8733-60a9-a4bb62cebabc'])) assert_equal([{ 'name': 'Dryrun', 'value': False }, { 'name': 'ls_arg', 'value': '-l' }], response.context['params_form'].initial) except Exception, ex: logging.exception(ex)
def import_workflow(request): if ENABLE_V2.get(): raise PopupException('/oozie/import_workflow is deprecated in the version 2 of Editor') workflow = Workflow.objects.new_workflow(request.user) if request.method == 'POST': workflow_form = ImportWorkflowForm(request.POST, request.FILES, instance=workflow) if workflow_form.is_valid(): if workflow_form.cleaned_data.get('resource_archive'): # Upload resources to workspace source = workflow_form.cleaned_data.get('resource_archive') if source.name.endswith('.zip'): workflow.save() Workflow.objects.initialize(workflow, request.fs) temp_path = archive_factory(source).extract() request.fs.copyFromLocal(temp_path, workflow.deployment_dir) shutil.rmtree(temp_path) else: raise PopupException(_('Archive should be a Zip.')) else: workflow.save() Workflow.objects.initialize(workflow, request.fs) workflow.managed = True workflow.save() workflow_definition = workflow_form.cleaned_data['definition_file'].read() try: _import_workflow(fs=request.fs, workflow=workflow, workflow_definition=workflow_definition) request.info(_('Workflow imported')) return redirect(reverse('oozie:edit_workflow', kwargs={'workflow': workflow.id})) except Exception, e: request.error(_('Could not import workflow: %s' % e)) Workflow.objects.destroy(workflow, request.fs) raise PopupException(_('Could not import workflow.'), detail=e) else: request.error(_('Errors on the form: %s') % workflow_form.errors)
def get_workflow(): if ENABLE_V2.get(): return Workflow else: return OldWorkflow
def get_history(): if ENABLE_V2.get(): return History else: return OldHistory
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) if ENABLE_V2.get(): # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.nodes else: workflow_graph, full_node_list = '', None else: history = get_history().cross_reference_submission_history(request.user, job_id) hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = Workflow.gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle) } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_workflow.id, 'parent_id': oozie_workflow.id } oozie_slas = oozie_api.get_oozie_slas(**params) return render('dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': parameters, 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph })
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = {} credentials = None doc = None hue_workflow = None hue_coord = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict) # When a workflow is submitted by a coordinator if not hue_workflow and hue_coord and hue_coord.workflow.document: hue_workflow = hue_coord.workflow if hue_coord and hue_coord.workflow and hue_coord.workflow.document: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() except Exception as e: LOG.exception("Error generating full page for running workflow %s with exception: %s" % (job_id, e.message)) finally: workflow_graph = '' credentials = Credentials() if not workflow_data.get('layout') or oozie_workflow.conf_dict.get('submit_single_action'): try: workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow) # Hide graph tab when node count > 30 if workflow_data.get('workflow') and len(workflow_data.get('workflow')['nodes']) > 30: workflow_data = {} except Exception as e: LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)) else: history = get_history().cross_reference_submission_history(request.user, job_id) hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = get_workflow().gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': if not workflow_graph and request.GET.get('is_jb2'): workflow_graph = django_mako.render_to_string('dashboard/list_oozie_workflow_graph.mako', {}) return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle), 'doc_url': doc.get_absolute_url() if doc else '', } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) if request.GET.get('format') == 'svg': oozie_api = get_oozie(request.user, api_version="v2") svg_data = oozie_api.get_job_graph(job_id) return HttpResponse(svg_data) if request.GET.get('graph'): return render('dashboard/list_oozie_workflow_graph.mako', request, { 'oozie_workflow': oozie_workflow, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data.get('workflow', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'graph_element_id': request.GET.get('element') if request.GET.get('element') else 'loaded ' + doc.uuid + ' graph', 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)), 'is_jb2': request.GET.get('is_jb2', False) }) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_workflow.id, 'parent_id': oozie_workflow.id } oozie_slas = oozie_api.get_oozie_slas(**params) return render('dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_parent': oozie_parent, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': dict((var, val) for var, val in parameters.items() if var not in ParameterForm.NON_PARAMETERS and var != 'oozie.use.system.libpath' or var == 'oozie.wf.application.path'), 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data.get('workflow', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)) })
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) if IS_HUE_4.get(): # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() elif USE_NEW_EDITOR.get(): # Install as link-workflow doc2 to old Job Designs docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) # Don't overwrite doc2, created = Document2.objects.get_or_create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data ) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) if not IS_HUE_4.get(): self.install_examples() Document.objects.sync()
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = None credentials = None doc = None hue_workflow = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: workflow_graph = '' full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() credentials = Credentials() else: # For workflows submitted from CLI or deleted in the editor # Until better parsing in https://issues.cloudera.org/browse/HUE-2659 workflow_graph, full_node_list = OldWorkflow.gen_status_graph_from_xml(request.user, oozie_workflow) except: LOG.exception("Error generating full page for running workflow %s" % job_id) else: history = get_history().cross_reference_submission_history(request.user, job_id) hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = get_workflow().gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle) } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_workflow.id, 'parent_id': oozie_workflow.id } oozie_slas = oozie_api.get_oozie_slas(**params) return render('dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_parent': oozie_parent, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': parameters, 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc1_id': doc.doc.get().id if doc else -1, 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)) })
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) if USE_NEW_EDITOR.get(): docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) doc2 = Document2.objects.create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission( request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission( request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = None credentials = None doc = None hue_workflow = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config( oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow ) or get_history().get_workflow_from_config( oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception( request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception( request.user) if hue_workflow: workflow_graph = '' full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = {'id': workflow_id} doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() credentials = Credentials() else: # For workflows submitted from CLI or deleted in the editor # Until better parsing in https://issues.cloudera.org/browse/HUE-2659 workflow_graph, full_node_list = OldWorkflow.gen_status_graph_from_xml( request.user, oozie_workflow) except: LOG.exception( "Error generating full page for running workflow %s" % job_id) else: history = get_history().cross_reference_submission_history( request.user, job_id) hue_coord = history and history.get_coordinator() or get_history( ).get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or ( history and history.get_workflow()) or get_history().get_workflow_from_config( oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = get_workflow( ).gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json( oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle) } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = {'id': oozie_workflow.id, 'parent_id': oozie_workflow.id} oozie_slas = oozie_api.get_oozie_slas(**params) return render( 'dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_parent': oozie_parent, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': parameters, 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc1_id': doc.doc.get().id if doc else -1, 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)) })
def handle(self, *args, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): with transaction.atomic(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2, commit=False) # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()])
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get("coordinator_job_id"): oozie_coordinator = check_job_access_permission(request, request.GET.get("coordinator_job_id")) oozie_bundle = None if request.GET.get("bundle_job_id"): oozie_bundle = check_job_access_permission(request, request.GET.get("bundle_job_id")) if oozie_coordinator is not None: setattr(oozie_workflow, "oozie_coordinator", oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, "oozie_bundle", oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = None credentials = None doc = None hue_workflow = None workflow_graph = "MISSING" # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or get_history().get_workflow_from_config( oozie_workflow.conf_dict ) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: workflow_graph = "" full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = {"id": workflow_id} doc = Document2.objects.get(type="oozie-workflow2", **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() credentials = Credentials() else: # For workflows submitted from CLI or deleted in the editor # Until better parsing in https://issues.cloudera.org/browse/HUE-2659 workflow_graph, full_node_list = OldWorkflow.gen_status_graph_from_xml(request.user, oozie_workflow) except: LOG.exception("Error generating full page for running workflow %s" % job_id) else: history = get_history().cross_reference_submission_history(request.user, job_id) hue_coord = ( history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict) ) hue_workflow = ( (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) ) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = get_workflow().gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get("format") == "json": return_obj = { "id": oozie_workflow.id, "status": oozie_workflow.status, "progress": oozie_workflow.get_progress(full_node_list), "graph": workflow_graph, "actions": massaged_workflow_actions_for_json( oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle ), } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = {"id": oozie_workflow.id, "parent_id": oozie_workflow.id} oozie_slas = oozie_api.get_oozie_slas(**params) return render( "dashboard/list_oozie_workflow.mako", request, { "oozie_workflow": oozie_workflow, "oozie_coordinator": oozie_coordinator, "oozie_bundle": oozie_bundle, "oozie_parent": oozie_parent, "oozie_slas": oozie_slas, "hue_workflow": hue_workflow, "hue_coord": hue_coord, "parameters": parameters, "has_job_edition_permission": has_job_edition_permission, "workflow_graph": workflow_graph, "layout_json": json.dumps(workflow_data["layout"], cls=JSONEncoderForHTML) if workflow_data else "", "workflow_json": json.dumps(workflow_data["workflow"], cls=JSONEncoderForHTML) if workflow_data else "", "credentials_json": json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML) if credentials else "", "workflow_properties_json": json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), "doc1_id": doc.doc.get().id if doc else -1, "subworkflows_json": json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), "can_edit_json": json.dumps(doc is None or doc.doc.get().is_editable(request.user)), }, )
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info( _("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info( _('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info( _('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir }) self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.copyFromLocal, local_dir, remote_data_dir) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR) if USE_NEW_EDITOR.get(): docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({ 'content_type': doc.content_type.model, 'object_id': doc.object_id }) data = json.dumps(data) doc2 = Document2.objects.create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data) LOG.info( 'Successfully installed sample link to jobsub: %s' % (doc2.name, )) # Share oozie examples with default group oozie_examples = Document2.objects.filter(type__in=[ 'oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2' ], owner=self.user, parent_directory=None) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) self.install_examples() Document.objects.sync()
def handle_noargs(self, **options): self.user = install_sample_user() self.fs = cluster.get_hdfs() LOG.info(_("Creating sample directory '%s' in HDFS") % REMOTE_SAMPLE_DIR.get()) create_directories(self.fs, [REMOTE_SAMPLE_DIR.get()]) remote_dir = REMOTE_SAMPLE_DIR.get() # Copy examples binaries for name in os.listdir(LOCAL_SAMPLE_DIR.get()): local_dir = self.fs.join(LOCAL_SAMPLE_DIR.get(), name) remote_data_dir = self.fs.join(remote_dir, name) LOG.info(_('Copying examples %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Copy sample data local_dir = LOCAL_SAMPLE_DATA_DIR.get() remote_data_dir = self.fs.join(remote_dir, 'data') LOG.info(_('Copying data %(local_dir)s to %(remote_data_dir)s\n') % { 'local_dir': local_dir, 'remote_data_dir': remote_data_dir}) self.fs.do_as_user(self.user.username, self.fs.copyFromLocal, local_dir, remote_data_dir) # Get or create sample user directories home_dir = Directory.objects.get_home_directory(self.user) examples_dir, created = Directory.objects.get_or_create( parent_directory=home_dir, owner=self.user, name=Document2.EXAMPLES_DIR ) # Load jobs LOG.info(_("Installing examples...")) if ENABLE_V2.get(): management.call_command('loaddata', 'initial_oozie_examples.json', verbosity=2) if IS_HUE_4.get(): # Install editor oozie examples without doc1 link LOG.info("Using Hue 4, will install oozie editor samples.") example_jobs = [] example_jobs.append(self._install_mapreduce_example()) example_jobs.append(self._install_java_example()) example_jobs.append(self._install_spark_example()) example_jobs.append(self._install_pyspark_example()) # If documents exist but have been trashed, recover from Trash for doc in example_jobs: if doc is not None and doc.parent_directory != examples_dir: doc.parent_directory = examples_dir doc.save() elif USE_NEW_EDITOR.get(): # Install as link-workflow doc2 to old Job Designs docs = Document.objects.get_docs(self.user, Workflow).filter(owner=self.user) for doc in docs: if doc.content_object: data = doc.content_object.data_dict data.update({'content_type': doc.content_type.model, 'object_id': doc.object_id}) data = json.dumps(data) # Don't overwrite doc2, created = Document2.objects.get_or_create( owner=self.user, parent_directory=examples_dir, name=doc.name, type='link-workflow', description=doc.description, data=data ) LOG.info('Successfully installed sample link to jobsub: %s' % (doc2.name,)) # Share oozie examples with default group oozie_examples = Document2.objects.filter( type__in=['oozie-workflow2', 'oozie-coordinator2', 'oozie-bundle2'], owner=self.user, parent_directory=None ) oozie_examples.update(parent_directory=examples_dir) examples_dir.share(self.user, Document2Permission.READ_PERM, groups=[get_default_user_group()]) if not IS_HUE_4.get(): self.install_examples() Document.objects.sync()
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission( request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission( request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) if ENABLE_V2.get(): # To update with the new History document model hue_coord = get_history().get_coordinator_from_config( oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow ) or get_history().get_workflow_from_config( oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception( request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.nodes else: workflow_graph, full_node_list = '', None else: history = get_history().cross_reference_submission_history( request.user, job_id) hue_coord = history and history.get_coordinator() or get_history( ).get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or ( history and history.get_workflow()) or get_history().get_workflow_from_config( oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = Workflow.gen_status_graph_from_xml( request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json( oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle) } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = {'id': oozie_workflow.id, 'parent_id': oozie_workflow.id} oozie_slas = oozie_api.get_oozie_slas(**params) return render( 'dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': parameters, 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph })