def test_gen_properties(self):
creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
metastore = {
'thrift_uri': 'thrift://hue-koh-chang:9999',
'kerberos_principal': 'hive',
}
finish = (
beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
)
try:
assert_equal({
'hcat': {
'xml_name': 'hcat',
'properties': [
('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
('hcat.metastore.principal', 'hive')
]},
'hive2': {
'xml_name': 'hive2',
'properties': [
('hive2.jdbc.url', 'jdbc:hive2://hue-koh-chang:12345/default'),
('hive2.server.principal', 'hive')
]},
'hbase': {
'xml_name': 'hbase',
'properties': []
}
}, creds.get_properties(metastore))
finally:
for f in finish:
f()
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({
'jobTracker': jobtracker_addr,
'nameNode': fs_defaultfs,
})
if self.job and deployment_dir:
self.properties.update({
self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties['credentials'] = credentials.get_properties()
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({
'jobTracker': jobtracker_addr,
'nameNode': fs_defaultfs,
})
if self.job and deployment_dir:
self.properties.update({
self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties['credentials'] = credentials.get_properties()
self._update_credentials_from_hive_action(credentials)
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info("Using FS %s and JT %s" % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({"jobTracker": jobtracker_addr, "nameNode": fs_defaultfs})
if self.job and deployment_dir:
self.properties.update(
{
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id,
}
)
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties["credentials"] = credentials.get_properties()
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
can_edit_json = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
return render(
'editor2/workflow_editor.mako', request, {
'layout_json':
json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML),
'workflow_json':
json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflow_properties_json':
json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'subworkflows_json':
json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json':
json.dumps(can_edit_json),
'is_embeddable':
request.GET.get('is_embeddable', False),
})
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def test_update_credentials_from_hive_action_when_jdbc_url_is_variable(
self):
class TestJob(object):
XML_FILE_NAME = 'workflow.xml'
def __init__(self):
self.deployment_dir = '/tmp/test'
self.nodes = [
Node({
'id': '1',
'type': 'hive-document',
'properties': {
'jdbc_url':
u"${wf:actionData('shell-31b5')['hiveserver']}",
'password': u'test'
}
})
]
user = User.objects.get(username='******')
submission = Submission(user,
job=TestJob(),
fs=MockFs(logical_name='fsname'),
jt=MockJt(logical_name='jtname'))
finish = (
beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
)
try:
creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
hive_properties = {
'thrift_uri': 'thrift://*****:*****@test-realm.com',
}
submission.properties['credentials'] = creds.get_properties(
hive_properties)
submission._update_credentials_from_hive_action(creds)
assert_equal(
submission.properties['credentials'][
creds.hiveserver2_name]['properties'],
[('hive2.jdbc.url',
u'jdbc:hive2://hue-koh-chang:12345/default'),
('hive2.server.principal', u'hive/[email protected]')
])
finally:
for f in finish:
f()
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(
type='oozie-workflow2',
owner=request.user,
is_managed=True,
dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user,
uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document,
user=request.user,
managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def test_parse_oozie(self):
oozie_credentialclasses = """
hbase=org.apache.oozie.action.hadoop.HbaseCredentials,
hcat=org.apache.oozie.action.hadoop.HCatCredentials,
hive2=org.apache.oozie.action.hadoop.Hive2Credentials
"""
oozie_config = {'oozie.credentials.credentialclasses': oozie_credentialclasses}
creds = Credentials()
assert_equal({
'hive2': 'org.apache.oozie.action.hadoop.Hive2Credentials',
'hbase': 'org.apache.oozie.action.hadoop.HbaseCredentials',
'hcat': 'org.apache.oozie.action.hadoop.HCatCredentials'
}, creds._parse_oozie(oozie_config)
)
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def list_oozie_workflow(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
oozie_coordinator = None
if request.GET.get('coordinator_job_id'):
oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id'))
oozie_bundle = None
if request.GET.get('bundle_job_id'):
oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id'))
if oozie_coordinator is not None:
setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator)
if oozie_bundle is not None:
setattr(oozie_workflow, 'oozie_bundle', oozie_bundle)
oozie_parent = oozie_workflow.get_parent_job_id()
if oozie_parent:
oozie_parent = check_job_access_permission(request, oozie_parent)
workflow_data = {}
credentials = None
doc = None
hue_workflow = None
hue_coord = None
workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows
full_node_list = None
if ENABLE_V2.get():
try:
# To update with the new History document model
hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict)
# When a workflow is submitted by a coordinator
if not hue_workflow and hue_coord:
hue_workflow = hue_coord.workflow
if hue_coord and hue_coord.workflow: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user)
if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user)
if hue_workflow:
full_node_list = hue_workflow.nodes
workflow_id = hue_workflow.id
wid = {
'id': workflow_id
}
doc = Document2.objects.get(type='oozie-workflow2', **wid)
new_workflow = get_workflow()(document=doc)
workflow_data = new_workflow.get_data()
if not workflow_data.get('layout'):
try:
workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow)
except Exception, e:
LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e))
workflow_graph = ''
credentials = Credentials()
except:
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def test_parse_oozie(self):
oozie_credentialclasses = """
hbase=org.apache.oozie.action.hadoop.HbaseCredentials,
hcat=org.apache.oozie.action.hadoop.HCatCredentials,
hive2=org.apache.oozie.action.hadoop.Hive2Credentials
"""
oozie_config = {
'oozie.credentials.credentialclasses': oozie_credentialclasses
}
creds = Credentials()
assert_equal(
{
'hive2': 'org.apache.oozie.action.hadoop.Hive2Credentials',
'hbase': 'org.apache.oozie.action.hadoop.HbaseCredentials',
'hcat': 'org.apache.oozie.action.hadoop.HCatCredentials'
}, creds._parse_oozie(oozie_config))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator')
doc = None
if coordinator_id:
doc = Document2.objects.get(id=coordinator_id)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def test_gen_properties(self):
creds = Credentials(credentials=TestCredentials.CREDENTIALS.copy())
hive_properties = {
'thrift_uri': 'thrift://hue-koh-chang:9999',
'kerberos_principal': 'hive',
'hive2.server.principal': 'hive',
}
finish = (
beeswax.conf.HIVE_SERVER_HOST.set_for_testing('hue-koh-chang'),
beeswax.conf.HIVE_SERVER_PORT.set_for_testing(12345),
)
try:
assert_equal(
{
'hcat': {
'xml_name':
'hcat',
'properties':
[('hcat.metastore.uri', 'thrift://hue-koh-chang:9999'),
('hcat.metastore.principal', 'hive')]
},
'hive2': {
'xml_name':
'hive2',
'properties':
[('hive2.jdbc.url',
'jdbc:hive2://hue-koh-chang:12345/default'),
('hive2.server.principal', 'hive')]
},
'hbase': {
'xml_name': 'hbase',
'properties': []
}
}, creds.get_properties(hive_properties))
finally:
for f in finish:
f()
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user,
job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.can_read(request.user)
user_can_edit_job = workflow.is_editable(request.user)
oozie_api = get_oozie(request.user)
credentials = Credentials()
credentials.fetch(oozie_api)
return render(
'editor/edit_workflow.mako', request, {
'oozie_api':
oozie_api,
'workflow_form':
workflow_form,
'workflow':
workflow,
'history':
history,
'user_can_access_job':
user_can_access_job,
'user_can_edit_job':
user_can_edit_job,
'job_properties':
extract_field_data(workflow_form['job_properties']),
'link_form':
LinkForm(),
'default_link_form':
DefaultLinkForm(action=workflow.start),
'node_form':
NodeForm(),
'action_forms':
[(node_type, design_form_by_type(node_type, request.user,
workflow)())
for node_type in ACTION_TYPES.iterkeys()],
'credentials':
json.dumps(credentials.credentials.keys())
})
def _update_properties(self, jobtracker_addr, deployment_dir):
if self.fs and self.jt:
self.properties.update(
{
"jobTracker": self.jt.logical_name or jobtracker_addr,
"nameNode": self.fs.logical_name or self.fs.fs_defaultfs,
}
)
if self.job:
self.properties.update(
{
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id,
}
)
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties["credentials"] = credentials.get_properties()
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user, job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.can_read(request.user)
user_can_edit_job = workflow.is_editable(request.user)
api = get_oozie(request.user)
credentials = Credentials()
credentials.fetch(api)
return render('editor/edit_workflow.mako', request, {
'workflow_form': workflow_form,
'workflow': workflow,
'history': history,
'user_can_access_job': user_can_access_job,
'user_can_edit_job': user_can_edit_job,
'job_properties': extract_field_data(workflow_form['job_properties']),
'link_form': LinkForm(),
'default_link_form': DefaultLinkForm(action=workflow.start),
'node_form': NodeForm(),
'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)())
for node_type in ACTION_TYPES.iterkeys()],
'credentials': json.dumps(credentials.credentials.keys())
})
def list_oozie_workflow(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
oozie_coordinator = None
if request.GET.get('coordinator_job_id'):
oozie_coordinator = check_job_access_permission(
request, request.GET.get('coordinator_job_id'))
oozie_bundle = None
if request.GET.get('bundle_job_id'):
oozie_bundle = check_job_access_permission(
request, request.GET.get('bundle_job_id'))
if oozie_coordinator is not None:
setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator)
if oozie_bundle is not None:
setattr(oozie_workflow, 'oozie_bundle', oozie_bundle)
oozie_parent = oozie_workflow.get_parent_job_id()
if oozie_parent:
oozie_parent = check_job_access_permission(request, oozie_parent)
workflow_data = None
credentials = None
doc = None
hue_workflow = None
workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows
full_node_list = None
if ENABLE_V2.get():
try:
# To update with the new History document model
hue_coord = get_history().get_coordinator_from_config(
oozie_workflow.conf_dict)
hue_workflow = (hue_coord and hue_coord.workflow
) or get_history().get_workflow_from_config(
oozie_workflow.conf_dict)
if hue_coord and hue_coord.workflow:
hue_coord.workflow.document.doc.get().can_read_or_exception(
request.user)
if hue_workflow:
hue_workflow.document.doc.get().can_read_or_exception(
request.user)
if hue_workflow:
workflow_graph = ''
full_node_list = hue_workflow.nodes
workflow_id = hue_workflow.id
wid = {'id': workflow_id}
doc = Document2.objects.get(type='oozie-workflow2', **wid)
new_workflow = get_workflow()(document=doc)
workflow_data = new_workflow.get_data()
credentials = Credentials()
else:
# For workflows submitted from CLI or deleted in the editor
# Until better parsing in https://issues.cloudera.org/browse/HUE-2659
workflow_graph, full_node_list = OldWorkflow.gen_status_graph_from_xml(
request.user, oozie_workflow)
except:
LOG.exception(
"Error generating full page for running workflow %s" % job_id)
else:
history = get_history().cross_reference_submission_history(
request.user, job_id)
hue_coord = history and history.get_coordinator() or get_history(
).get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = (hue_coord and hue_coord.workflow) or (
history and
history.get_workflow()) or get_history().get_workflow_from_config(
oozie_workflow.conf_dict)
if hue_coord and hue_coord.workflow:
Job.objects.can_read_or_exception(request, hue_coord.workflow.id)
if hue_workflow:
Job.objects.can_read_or_exception(request, hue_workflow.id)
if hue_workflow:
workflow_graph = hue_workflow.gen_status_graph(oozie_workflow)
full_node_list = hue_workflow.node_list
else:
workflow_graph, full_node_list = get_workflow(
).gen_status_graph_from_xml(request.user, oozie_workflow)
parameters = oozie_workflow.conf_dict.copy()
for action in oozie_workflow.actions:
action.oozie_coordinator = oozie_coordinator
action.oozie_bundle = oozie_bundle
if request.GET.get('format') == 'json':
return_obj = {
'id':
oozie_workflow.id,
'status':
oozie_workflow.status,
'progress':
oozie_workflow.get_progress(full_node_list),
'graph':
workflow_graph,
'actions':
massaged_workflow_actions_for_json(
oozie_workflow.get_working_actions(), oozie_coordinator,
oozie_bundle)
}
return JsonResponse(return_obj, encoder=JSONEncoderForHTML)
oozie_slas = []
if oozie_workflow.has_sla:
oozie_api = get_oozie(request.user, api_version="v2")
params = {'id': oozie_workflow.id, 'parent_id': oozie_workflow.id}
oozie_slas = oozie_api.get_oozie_slas(**params)
return render(
'dashboard/list_oozie_workflow.mako', request, {
'oozie_workflow':
oozie_workflow,
'oozie_coordinator':
oozie_coordinator,
'oozie_bundle':
oozie_bundle,
'oozie_parent':
oozie_parent,
'oozie_slas':
oozie_slas,
'hue_workflow':
hue_workflow,
'hue_coord':
hue_coord,
'parameters':
parameters,
'has_job_edition_permission':
has_job_edition_permission,
'workflow_graph':
workflow_graph,
'layout_json':
json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML)
if workflow_data else '',
'workflow_json':
json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML)
if workflow_data else '',
'credentials_json':
json.dumps(credentials.credentials.keys(), cls=JSONEncoderForHTML)
if credentials else '',
'workflow_properties_json':
json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc1_id':
doc.doc.get().id if doc else -1,
'subworkflows_json':
json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json':
json.dumps(doc is None or doc.doc.get().is_editable(request.user))
})
def list_oozie_workflow(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
oozie_coordinator = None
if request.GET.get('coordinator_job_id'):
oozie_coordinator = check_job_access_permission(request, request.GET.get('coordinator_job_id'))
oozie_bundle = None
if request.GET.get('bundle_job_id'):
oozie_bundle = check_job_access_permission(request, request.GET.get('bundle_job_id'))
if oozie_coordinator is not None:
setattr(oozie_workflow, 'oozie_coordinator', oozie_coordinator)
if oozie_bundle is not None:
setattr(oozie_workflow, 'oozie_bundle', oozie_bundle)
oozie_parent = oozie_workflow.get_parent_job_id()
if oozie_parent:
oozie_parent = check_job_access_permission(request, oozie_parent)
workflow_data = {}
credentials = None
doc = None
hue_workflow = None
hue_coord = None
workflow_graph = 'MISSING' # default to prevent loading the graph tab for deleted workflows
full_node_list = None
if ENABLE_V2.get():
try:
# To update with the new History document model
hue_coord = get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = get_history().get_workflow_from_config(oozie_workflow.conf_dict)
# When a workflow is submitted by a coordinator
if not hue_workflow and hue_coord and hue_coord.workflow.document:
hue_workflow = hue_coord.workflow
if hue_coord and hue_coord.workflow and hue_coord.workflow.document: hue_coord.workflow.document.doc.get().can_read_or_exception(request.user)
if hue_workflow: hue_workflow.document.doc.get().can_read_or_exception(request.user)
if hue_workflow:
full_node_list = hue_workflow.nodes
workflow_id = hue_workflow.id
wid = {
'id': workflow_id
}
doc = Document2.objects.get(type='oozie-workflow2', **wid)
new_workflow = get_workflow()(document=doc)
workflow_data = new_workflow.get_data()
except Exception as e:
LOG.exception("Error generating full page for running workflow %s with exception: %s" % (job_id, e.message))
finally:
workflow_graph = ''
credentials = Credentials()
if not workflow_data.get('layout') or oozie_workflow.conf_dict.get('submit_single_action'):
try:
workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow)
# Hide graph tab when node count > 30
if workflow_data.get('workflow') and len(workflow_data.get('workflow')['nodes']) > 30:
workflow_data = {}
except Exception as e:
LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e))
else:
history = get_history().cross_reference_submission_history(request.user, job_id)
hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict)
if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id)
if hue_workflow: Job.objects.can_read_or_exception(request, hue_workflow.id)
if hue_workflow:
workflow_graph = hue_workflow.gen_status_graph(oozie_workflow)
full_node_list = hue_workflow.node_list
else:
workflow_graph, full_node_list = get_workflow().gen_status_graph_from_xml(request.user, oozie_workflow)
parameters = oozie_workflow.conf_dict.copy()
for action in oozie_workflow.actions:
action.oozie_coordinator = oozie_coordinator
action.oozie_bundle = oozie_bundle
if request.GET.get('format') == 'json':
if not workflow_graph and request.GET.get('is_jb2'):
workflow_graph = django_mako.render_to_string('dashboard/list_oozie_workflow_graph.mako', {})
return_obj = {
'id': oozie_workflow.id,
'status': oozie_workflow.status,
'progress': oozie_workflow.get_progress(full_node_list),
'graph': workflow_graph,
'actions': massaged_workflow_actions_for_json(oozie_workflow.get_working_actions(), oozie_coordinator, oozie_bundle),
'doc_url': doc.get_absolute_url() if doc else '',
}
return JsonResponse(return_obj, encoder=JSONEncoderForHTML)
if request.GET.get('format') == 'svg':
oozie_api = get_oozie(request.user, api_version="v2")
svg_data = oozie_api.get_job_graph(job_id)
return HttpResponse(svg_data)
if request.GET.get('graph'):
return render('dashboard/list_oozie_workflow_graph.mako', request, {
'oozie_workflow': oozie_workflow,
'workflow_graph': workflow_graph,
'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '',
'workflow_json': json.dumps(workflow_data.get('workflow', ''), cls=JSONEncoderForHTML) if workflow_data else '',
'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML) if credentials else '',
'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'graph_element_id': request.GET.get('element') if request.GET.get('element') else 'loaded ' + doc.uuid + ' graph',
'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user)),
'is_jb2': request.GET.get('is_jb2', False)
})
oozie_slas = []
if oozie_workflow.has_sla:
oozie_api = get_oozie(request.user, api_version="v2")
params = {
'id': oozie_workflow.id,
'parent_id': oozie_workflow.id
}
oozie_slas = oozie_api.get_oozie_slas(**params)
return render('dashboard/list_oozie_workflow.mako', request, {
'oozie_workflow': oozie_workflow,
'oozie_coordinator': oozie_coordinator,
'oozie_bundle': oozie_bundle,
'oozie_parent': oozie_parent,
'oozie_slas': oozie_slas,
'hue_workflow': hue_workflow,
'hue_coord': hue_coord,
'parameters': dict((var, val) for var, val in parameters.items() if var not in ParameterForm.NON_PARAMETERS and var != 'oozie.use.system.libpath' or var == 'oozie.wf.application.path'),
'has_job_edition_permission': has_job_edition_permission,
'workflow_graph': workflow_graph,
'layout_json': json.dumps(workflow_data.get('layout', ''), cls=JSONEncoderForHTML) if workflow_data else '',
'workflow_json': json.dumps(workflow_data.get('workflow', ''), cls=JSONEncoderForHTML) if workflow_data else '',
'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML) if credentials else '',
'workflow_properties_json': json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc_uuid': doc.uuid if doc else '',
'subworkflows_json': json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json': json.dumps(doc is None or doc.doc.get().is_editable(request.user))
})
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
scheduled_uuid = coordinator.data['properties'][
'workflow'] or coordinator.data['properties']['document']
if scheduled_uuid:
try:
document = Document2.objects.get(uuid=scheduled_uuid)
except Document2.DoesNotExist as e:
document = None
coordinator.data['properties']['workflow'] = ''
LOG.warning("Workflow with uuid %s doesn't exist: %s" %
(scheduled_uuid, e))
if document and document.is_trashed:
raise PopupException(
_('Your workflow %s has been trashed!') %
(document.name if document.name else ''))
if document and not document.can_read(request.user):
raise PopupException(
_('You don\'t have access to the workflow or document of this coordinator.'
))
else:
workflows = [
dict([('uuid', d.content_object.uuid),
('name', d.content_object.name)])
for d in Document.objects.available_docs(
Document2, request.user).filter(extra='workflow2')
]
if coordinator_id and not [
a for a in workflows
if a['uuid'] == coordinator.data['properties']['workflow']
]:
raise PopupException(
_('You don\'t have access to the workflow of this coordinator.'
))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [
dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(
request.user, include_managed=False).search_documents(
types=['oozie-workflow2'])
]
can_edit = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator':
coordinator.get_data_for_json(),
'credentials':
list(credentials.credentials.keys()),
'workflows':
workflows,
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit':
can_edit,
'layout':
django_mako.render_to_string(
'editor2/common_scheduler.mako',
{'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render(
'editor2/coordinator_editor.mako', request, {
'coordinator_json':
coordinator.to_json_for_html(),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflows_json':
json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_json':
json.dumps(can_edit)
})
if hue_workflow:
full_node_list = hue_workflow.nodes
workflow_id = hue_workflow.id
wid = {
'id': workflow_id
}
doc = Document2.objects.get(type='oozie-workflow2', **wid)
new_workflow = get_workflow()(document=doc)
workflow_data = new_workflow.get_data()
except Exception, e:
LOG.exception("Error generating full page for running workflow %s with exception: %s" % (job_id, e.message))
finally:
workflow_graph = ''
credentials = Credentials()
if not workflow_data.get('layout') or oozie_workflow.conf_dict.get('submit_single_action'):
try:
workflow_data = Workflow.gen_workflow_data_from_xml(request.user, oozie_workflow)
# Hide graph tab when node count > 30
if workflow_data.get('workflow') and len(workflow_data.get('workflow')['nodes']) > 30:
workflow_data = {}
except Exception, e:
LOG.exception('Graph data could not be generated from Workflow %s: %s' % (oozie_workflow.id, e))
else:
history = get_history().cross_reference_submission_history(request.user, job_id)
hue_coord = history and history.get_coordinator() or get_history().get_coordinator_from_config(oozie_workflow.conf_dict)
hue_workflow = (hue_coord and hue_coord.workflow) or (history and history.get_workflow()) or get_history().get_workflow_from_config(oozie_workflow.conf_dict)
if hue_coord and hue_coord.workflow: Job.objects.can_read_or_exception(request, hue_coord.workflow.id)
