def test_gen_properties(self): creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy()) metastore = { 'thrift_uri': 'thrift://hue-koh-chang:9999', 'kerberos_principal': 'hive', } finish = ( beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'), beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345), ) try: assert_equal({ 'hcat': { 'xml_name': 'hcat', 'properties': [ ('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'), ('hcat.metastore.principal', 'hive') ]}, 'hive2': { 'xml_name': 'hive2', 'properties': [ ('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'), ('hive2.server.principal', 'hive') ]}, 'hbase': { 'xml_name': 'hbase', 'properties': [] } }, creds.get_properties(metastore)) finally: for f in finish: f()
def _update_properties(self, jobtracker_addr, deployment_dir=None): LOG.info('Using FS %s and JT %s' % (self.fs, self.jt)) if self.jt and self.jt.logical_name: jobtracker_addr = self.jt.logical_name if self.fs.logical_name: fs_defaultfs = self.fs.logical_name else: fs_defaultfs = self.fs.fs_defaultfs self.properties.update({ 'jobTracker': jobtracker_addr, 'nameNode': fs_defaultfs, }) if self.job and deployment_dir: self.properties.update({ self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id }) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties['credentials'] = credentials.get_properties()
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) if request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def _update_properties(self, jobtracker_addr, deployment_dir=None): LOG.info('Using FS %s and JT %s' % (self.fs, self.jt)) if self.jt and self.jt.logical_name: jobtracker_addr = self.jt.logical_name if self.fs.logical_name: fs_defaultfs = self.fs.logical_name else: fs_defaultfs = self.fs.fs_defaultfs self.properties.update({ 'jobTracker': jobtracker_addr, 'nameNode': fs_defaultfs, }) if self.job and deployment_dir: self.properties.update({ self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id }) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties['credentials'] = credentials.get_properties() self._update_credentials_from_hive_action(credentials)
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def _update_properties(self, jobtracker_addr, deployment_dir=None): LOG.info("Using FS %s and JT %s" % (self.fs, self.jt)) if self.jt and self.jt.logical_name: jobtracker_addr = self.jt.logical_name if self.fs.logical_name: fs_defaultfs = self.fs.logical_name else: fs_defaultfs = self.fs.fs_defaultfs self.properties.update({"jobTracker": jobtracker_addr, "nameNode": fs_defaultfs}) if self.job and deployment_dir: self.properties.update( { self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id, } ) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties["credentials"] = credentials.get_properties()
def _edit_workflow(request, doc, workflow): workflow_data = workflow.get_data() api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception as e: LOG.error(smart_str(e)) can_edit_json = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user)) return render( 'editor2/workflow_editor.mako', request, { 'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML), 'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML), 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML), 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(can_edit_json), 'is_embeddable': request.GET.get('is_embeddable', False), })
def _edit_workflow(request, doc, workflow): workflow_data = workflow.get_data() api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def test_update_credentials_from_hive_action_when_jdbc_url_is_variable( self): class TestJob(object): XML_FILE_NAME = 'workflow.xml' def __init__(self): self.deployment_dir = '/tmp/test' self.nodes = [ Node({ 'id': '1', 'type': 'hive-document', 'properties': { 'jdbc_url': u"${wf:actionData('shell-31b5')['hiveserver']}", 'password': u'test' } }) ] user = User.objects.get(username='******') submission = Submission(user, job=TestJob(), fs=MockFs(logical_name='fsname'), jt=MockJt(logical_name='jtname')) finish = ( beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'), beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345), ) try: creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy()) hive_properties = { 'thrift_uri': 'thrift://*****:*****@test-realm.com', } submission.properties['credentials'] = creds.get_properties( hive_properties) submission._update_credentials_from_hive_action(creds) assert_equal( submission.properties['credentials'][ creds.hiveserver2_name]['properties'], [('hive2.jdbc.url', u'jdbc:hive2://hue-koh-chang:12345/default'), ('hive2.server.principal', u'hive/[email protected]') ]) finally: for f in finish: f()
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) # Automatically create the workflow of a scheduled document # To move to save coordinator document_uuid = request.GET.get('document') if document_uuid: # Has already a workflow managing the query for this user? workflows = Document2.objects.filter( type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid]) if workflows.exists(): workflow_doc = workflows.get() else: document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid) workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True) if doc: doc.dependencies.add(workflow_doc) workflow_uuid = workflow_doc.uuid coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name elif request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def test_parse_oozie(self): oozie_credentialclasses = """ hbase=org.apache.oozie.action.hadoop.HbaseCredentials, hcat=org.apache.oozie.action.hadoop.HCatCredentials, hive2=org.apache.oozie.action.hadoop.Hive2Credentials """ oozie_config = {'oozie.credentials.credentialclasses': oozie_credentialclasses} creds = Credentials() assert_equal({ 'hive2': 'org.apache.oozie.action.hadoop.Hive2Credentials', 'hbase': 'org.apache.oozie.action.hadoop.HbaseCredentials', 'hcat': 'org.apache.oozie.action.hadoop.HCatCredentials' }, creds._parse_oozie(oozie_config) )
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = {} credentials = None doc = None hue_workflow = None hue_coord = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict) # When a workflow is submitted by a coordinator if not hue_workflow and hue_coord: hue_workflow = hue_coord.workflow if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() if not workflow_data.get('layout'): try: workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow) except Exception, e: LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)) workflow_graph = '' credentials = Credentials() except:
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) # Automatically create the workflow of a scheduled document # To move to save coordinator document_uuid = request.GET.get('document') if document_uuid: # Has already a workflow managing the query for this user? workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid]) if workflows.exists(): workflow_doc = workflows.get() else: document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid) workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True) if doc: doc.dependencies.add(workflow_doc) workflow_uuid = workflow_doc.uuid coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name elif request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def test_parse_oozie(self): oozie_credentialclasses = """ hbase=org.apache.oozie.action.hadoop.HbaseCredentials, hcat=org.apache.oozie.action.hadoop.HCatCredentials, hive2=org.apache.oozie.action.hadoop.Hive2Credentials """ oozie_config = { 'oozie.credentials.credentialclasses': oozie_credentialclasses } creds = Credentials() assert_equal( { 'hive2': 'org.apache.oozie.action.hadoop.Hive2Credentials', 'hbase': 'org.apache.oozie.action.hadoop.HbaseCredentials', 'hcat': 'org.apache.oozie.action.hadoop.HCatCredentials' }, creds._parse_oozie(oozie_config))
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator') doc = None if coordinator_id: doc = Document2.objects.get(id=coordinator_id) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def test_gen_properties(self): creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy()) hive_properties = { 'thrift_uri': 'thrift://hue-koh-chang:9999', 'kerberos_principal': 'hive', 'hive2.server.principal': 'hive', } finish = ( beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'), beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345), ) try: assert_equal( { 'hcat': { 'xml_name': 'hcat', 'properties': [('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'), ('hcat.metastore.principal', 'hive')] }, 'hive2': { 'xml_name': 'hive2', 'properties': [('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'), ('hive2.server.principal', 'hive')] }, 'hbase': { 'xml_name': 'hbase', 'properties': [] } }, creds.get_properties(hive_properties)) finally: for f in finish: f()
def edit_workflow(request, workflow): history = History.objects.filter(submitter=request.user, job=workflow).order_by('-submission_date') workflow_form = WorkflowForm(instance=workflow) user_can_access_job = workflow.can_read(request.user) user_can_edit_job = workflow.is_editable(request.user) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) return render( 'editor/edit_workflow.mako', request, { 'oozie_api': oozie_api, 'workflow_form': workflow_form, 'workflow': workflow, 'history': history, 'user_can_access_job': user_can_access_job, 'user_can_edit_job': user_can_edit_job, 'job_properties': extract_field_data(workflow_form['job_properties']), 'link_form': LinkForm(), 'default_link_form': DefaultLinkForm(action=workflow.start), 'node_form': NodeForm(), 'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)()) for node_type in ACTION_TYPES.iterkeys()], 'credentials': json.dumps(credentials.credentials.keys()) })
def _update_properties(self, jobtracker_addr, deployment_dir): if self.fs and self.jt: self.properties.update( { "jobTracker": self.jt.logical_name or jobtracker_addr, "nameNode": self.fs.logical_name or self.fs.fs_defaultfs, } ) if self.job: self.properties.update( { self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir), self.job.HUE_ID: self.job.id, } ) # Generate credentials when using security if self.api.security_enabled: credentials = Credentials() credentials.fetch(self.api) self.properties["credentials"] = credentials.get_properties()
def edit_workflow(request, workflow): history = History.objects.filter(submitter=request.user, job=workflow).order_by('-submission_date') workflow_form = WorkflowForm(instance=workflow) user_can_access_job = workflow.can_read(request.user) user_can_edit_job = workflow.is_editable(request.user) api = get_oozie(request.user) credentials = Credentials() credentials.fetch(api) return render('editor/edit_workflow.mako', request, { 'workflow_form': workflow_form, 'workflow': workflow, 'history': history, 'user_can_access_job': user_can_access_job, 'user_can_edit_job': user_can_edit_job, 'job_properties': extract_field_data(workflow_form['job_properties']), 'link_form': LinkForm(), 'default_link_form': DefaultLinkForm(action=workflow.start), 'node_form': NodeForm(), 'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)()) for node_type in ACTION_TYPES.iterkeys()], 'credentials': json.dumps(credentials.credentials.keys()) })
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission( request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission( request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = None credentials = None doc = None hue_workflow = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config( oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow ) or get_history().get_workflow_from_config( oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception( request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception( request.user) if hue_workflow: workflow_graph = '' full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = {'id': workflow_id} doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() credentials = Credentials() else: # For workflows submitted from CLI or deleted in the editor # Until better parsing in https://issues.cloudera.org/browse/HUE-2659 workflow_graph, full_node_list = OldWorkflow.gen_status_graph_from_xml( request.user, oozie_workflow) except: LOG.exception( "Error generating full page for running workflow %s" % job_id) else: history = get_history().cross_reference_submission_history( request.user, job_id) hue_coord = history and history.get_coordinator() or get_history( ).get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or ( history and history.get_workflow()) or get_history().get_workflow_from_config( oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = get_workflow( ).gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json( oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle) } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = {'id': oozie_workflow.id, 'parent_id': oozie_workflow.id} oozie_slas = oozie_api.get_oozie_slas(**params) return render( 'dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_parent': oozie_parent, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': parameters, 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc1_id': doc.doc.get().id if doc else -1, 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)) })
def list_oozie_workflow(request, job_id): oozie_workflow = check_job_access_permission(request, job_id) oozie_coordinator = None if request.GET.get('coordinator_job_id'): oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id')) oozie_bundle = None if request.GET.get('bundle_job_id'): oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id')) if oozie_coordinator is not None: setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator) if oozie_bundle is not None: setattr(oozie_workflow, 'oozie_bundle', oozie_bundle) oozie_parent = oozie_workflow.get_parent_job_id() if oozie_parent: oozie_parent = check_job_access_permission(request, oozie_parent) workflow_data = {} credentials = None doc = None hue_workflow = None hue_coord = None workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows full_node_list = None if ENABLE_V2.get(): try: # To update with the new History document model hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict) # When a workflow is submitted by a coordinator if not hue_workflow and hue_coord and hue_coord.workflow.document: hue_workflow = hue_coord.workflow if hue_coord and hue_coord.workflow and hue_coord.workflow.document: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user) if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() except Exception as e: LOG.exception("Error generating full page for running workflow %s with exception: %s" % (job_id, e.message)) finally: workflow_graph = '' credentials = Credentials() if not workflow_data.get('layout') or oozie_workflow.conf_dict.get('submit_single_action'): try: workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow) # Hide graph tab when node count > 30 if workflow_data.get('workflow') and len(workflow_data.get('workflow')['nodes']) > 30: workflow_data = {} except Exception as e: LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)) else: history = get_history().cross_reference_submission_history(request.user, job_id) hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id) if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id) if hue_workflow: workflow_graph = hue_workflow.gen_status_graph(oozie_workflow) full_node_list = hue_workflow.node_list else: workflow_graph, full_node_list = get_workflow().gen_status_graph_from_xml(request.user, oozie_workflow) parameters = oozie_workflow.conf_dict.copy() for action in oozie_workflow.actions: action.oozie_coordinator = oozie_coordinator action.oozie_bundle = oozie_bundle if request.GET.get('format') == 'json': if not workflow_graph and request.GET.get('is_jb2'): workflow_graph = django_mako.render_to_string('dashboard/list_oozie_workflow_graph.mako', {}) return_obj = { 'id': oozie_workflow.id, 'status': oozie_workflow.status, 'progress': oozie_workflow.get_progress(full_node_list), 'graph': workflow_graph, 'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle), 'doc_url': doc.get_absolute_url() if doc else '', } return JsonResponse(return_obj, encoder=JSONEncoderForHTML) if request.GET.get('format') == 'svg': oozie_api = get_oozie(request.user, api_version="v2") svg_data = oozie_api.get_job_graph(job_id) return HttpResponse(svg_data) if request.GET.get('graph'): return render('dashboard/list_oozie_workflow_graph.mako', request, { 'oozie_workflow': oozie_workflow, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data.get('workflow', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'graph_element_id': request.GET.get('element') if request.GET.get('element') else 'loaded ' + doc.uuid + ' graph', 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)), 'is_jb2': request.GET.get('is_jb2', False) }) oozie_slas = [] if oozie_workflow.has_sla: oozie_api = get_oozie(request.user, api_version="v2") params = { 'id': oozie_workflow.id, 'parent_id': oozie_workflow.id } oozie_slas = oozie_api.get_oozie_slas(**params) return render('dashboard/list_oozie_workflow.mako', request, { 'oozie_workflow': oozie_workflow, 'oozie_coordinator': oozie_coordinator, 'oozie_bundle': oozie_bundle, 'oozie_parent': oozie_parent, 'oozie_slas': oozie_slas, 'hue_workflow': hue_workflow, 'hue_coord': hue_coord, 'parameters': dict((var, val) for var, val in parameters.items() if var not in ParameterForm.NON_PARAMETERS and var != 'oozie.use.system.libpath' or var == 'oozie.wf.application.path'), 'has_job_edition_permission': has_job_edition_permission, 'workflow_graph': workflow_graph, 'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'workflow_json': json.dumps(workflow_data.get('workflow', ''), cls=JSONEncoderForHTML) if workflow_data else '', 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML) if credentials else '', 'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML), 'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)) })
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) if request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception as e: LOG.error(smart_str(e)) if USE_NEW_EDITOR.get(): scheduled_uuid = coordinator.data['properties'][ 'workflow'] or coordinator.data['properties']['document'] if scheduled_uuid: try: document = Document2.objects.get(uuid=scheduled_uuid) except Document2.DoesNotExist as e: document = None coordinator.data['properties']['workflow'] = '' LOG.warning("Workflow with uuid %s doesn't exist: %s" % (scheduled_uuid, e)) if document and document.is_trashed: raise PopupException( _('Your workflow %s has been trashed!') % (document.name if document.name else '')) if document and not document.can_read(request.user): raise PopupException( _('You don\'t have access to the workflow or document of this coordinator.' )) else: workflows = [ dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)]) for d in Document.objects.available_docs( Document2, request.user).filter(extra='workflow2') ] if coordinator_id and not [ a for a in workflows if a['uuid'] == coordinator.data['properties']['workflow'] ]: raise PopupException( _('You don\'t have access to the workflow of this coordinator.' )) if USE_NEW_EDITOR.get(): # In Hue 4, merge with above workflows = [ dict([('uuid', d.uuid), ('name', d.name)]) for d in Document2.objects.documents( request.user, include_managed=False).search_documents( types=['oozie-workflow2']) ] can_edit = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user)) if request.GET.get('format') == 'json': # For Editor return JsonResponse({ 'coordinator': coordinator.get_data_for_json(), 'credentials': list(credentials.credentials.keys()), 'workflows': workflows, 'doc_uuid': doc.uuid if doc else '', 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit': can_edit, 'layout': django_mako.render_to_string( 'editor2/common_scheduler.mako', {'coordinator_json': coordinator.to_json_for_html()}) }) else: return render( 'editor2/coordinator_editor.mako', request, { 'coordinator_json': coordinator.to_json_for_html(), 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML), 'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit_json': json.dumps(can_edit) })
if hue_workflow: full_node_list = hue_workflow.nodes workflow_id = hue_workflow.id wid = { 'id': workflow_id } doc = Document2.objects.get(type='oozie-workflow2', **wid) new_workflow = get_workflow()(document=doc) workflow_data = new_workflow.get_data() except Exception, e: LOG.exception("Error generating full page for running workflow %s with exception: %s" % (job_id, e.message)) finally: workflow_graph = '' credentials = Credentials() if not workflow_data.get('layout') or oozie_workflow.conf_dict.get('submit_single_action'): try: workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow) # Hide graph tab when node count > 30 if workflow_data.get('workflow') and len(workflow_data.get('workflow')['nodes']) > 30: workflow_data = {} except Exception, e: LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e)) else: history = get_history().cross_reference_submission_history(request.user, job_id) hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict) hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict) if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id)