def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({
'jobTracker': jobtracker_addr,
'nameNode': fs_defaultfs,
})
if self.job and deployment_dir:
self.properties.update({
self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties['credentials'] = credentials.get_properties()
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info('Using FS %s and JT %s' % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({
'jobTracker': jobtracker_addr,
'nameNode': fs_defaultfs,
})
if self.job and deployment_dir:
self.properties.update({
self.job.PROPERTY_APP_PATH: self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id
})
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties['credentials'] = credentials.get_properties()
self._update_credentials_from_hive_action(credentials)
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
can_edit_json = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
return render(
'editor2/workflow_editor.mako', request, {
'layout_json':
json.dumps(workflow_data['layout'], cls=JSONEncoderForHTML),
'workflow_json':
json.dumps(workflow_data['workflow'], cls=JSONEncoderForHTML),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflow_properties_json':
json.dumps(WORKFLOW_NODE_PROPERTIES, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'subworkflows_json':
json.dumps(_get_workflows(request.user), cls=JSONEncoderForHTML),
'can_edit_json':
json.dumps(can_edit_json),
'is_embeddable':
request.GET.get('is_embeddable', False),
})
def _update_properties(self, jobtracker_addr, deployment_dir=None):
LOG.info("Using FS %s and JT %s" % (self.fs, self.jt))
if self.jt and self.jt.logical_name:
jobtracker_addr = self.jt.logical_name
if self.fs.logical_name:
fs_defaultfs = self.fs.logical_name
else:
fs_defaultfs = self.fs.fs_defaultfs
self.properties.update({"jobTracker": jobtracker_addr, "nameNode": fs_defaultfs})
if self.job and deployment_dir:
self.properties.update(
{
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id,
}
)
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties["credentials"] = credentials.get_properties()
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _edit_workflow(request, doc, workflow):
workflow_data = workflow.get_data()
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(
type='oozie-workflow2',
owner=request.user,
is_managed=True,
dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user,
uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document,
user=request.user,
managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def export_workflow(request, workflow): mapping = dict([(param['name'], param['value']) for param in workflow.find_all_parameters()]) oozie_api = get_oozie(request.user) credentials = Credentials() credentials.fetch(oozie_api) mapping['credentials'] = credentials.get_properties() zip_file = workflow.compress(mapping=mapping) response = HttpResponse(content_type="application/zip") response["Last-Modified"] = http_date(time.time()) response["Content-Length"] = len(zip_file.getvalue()) response['Content-Disposition'] = 'attachment; filename="workflow-%s-%d.zip"' % (workflow.name, workflow.id) response.write(zip_file.getvalue()) return response
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator')
doc = None
if coordinator_id:
doc = Document2.objects.get(id=coordinator_id)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def _update_properties(self, jobtracker_addr, deployment_dir):
if self.fs and self.jt:
self.properties.update(
{
"jobTracker": self.jt.logical_name or jobtracker_addr,
"nameNode": self.fs.logical_name or self.fs.fs_defaultfs,
}
)
if self.job:
self.properties.update(
{
self.job.get_application_path_key(): self.fs.get_hdfs_path(deployment_dir),
self.job.HUE_ID: self.job.id,
}
)
# Generate credentials when using security
if self.api.security_enabled:
credentials = Credentials()
credentials.fetch(self.api)
self.properties["credentials"] = credentials.get_properties()
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user,
job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.can_read(request.user)
user_can_edit_job = workflow.is_editable(request.user)
oozie_api = get_oozie(request.user)
credentials = Credentials()
credentials.fetch(oozie_api)
return render(
'editor/edit_workflow.mako', request, {
'oozie_api':
oozie_api,
'workflow_form':
workflow_form,
'workflow':
workflow,
'history':
history,
'user_can_access_job':
user_can_access_job,
'user_can_edit_job':
user_can_edit_job,
'job_properties':
extract_field_data(workflow_form['job_properties']),
'link_form':
LinkForm(),
'default_link_form':
DefaultLinkForm(action=workflow.start),
'node_form':
NodeForm(),
'action_forms':
[(node_type, design_form_by_type(node_type, request.user,
workflow)())
for node_type in ACTION_TYPES.iterkeys()],
'credentials':
json.dumps(credentials.credentials.keys())
})
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user, job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.can_read(request.user)
user_can_edit_job = workflow.is_editable(request.user)
api = get_oozie(request.user)
credentials = Credentials()
credentials.fetch(api)
return render('editor/edit_workflow.mako', request, {
'workflow_form': workflow_form,
'workflow': workflow,
'history': history,
'user_can_access_job': user_can_access_job,
'user_can_edit_job': user_can_edit_job,
'job_properties': extract_field_data(workflow_form['job_properties']),
'link_form': LinkForm(),
'default_link_form': DefaultLinkForm(action=workflow.start),
'node_form': NodeForm(),
'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)())
for node_type in ACTION_TYPES.iterkeys()],
'credentials': json.dumps(credentials.credentials.keys())
})
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
scheduled_uuid = coordinator.data['properties'][
'workflow'] or coordinator.data['properties']['document']
if scheduled_uuid:
try:
document = Document2.objects.get(uuid=scheduled_uuid)
except Document2.DoesNotExist as e:
document = None
coordinator.data['properties']['workflow'] = ''
LOG.warning("Workflow with uuid %s doesn't exist: %s" %
(scheduled_uuid, e))
if document and document.is_trashed:
raise PopupException(
_('Your workflow %s has been trashed!') %
(document.name if document.name else ''))
if document and not document.can_read(request.user):
raise PopupException(
_('You don\'t have access to the workflow or document of this coordinator.'
))
else:
workflows = [
dict([('uuid', d.content_object.uuid),
('name', d.content_object.name)])
for d in Document.objects.available_docs(
Document2, request.user).filter(extra='workflow2')
]
if coordinator_id and not [
a for a in workflows
if a['uuid'] == coordinator.data['properties']['workflow']
]:
raise PopupException(
_('You don\'t have access to the workflow of this coordinator.'
))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [
dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(
request.user, include_managed=False).search_documents(
types=['oozie-workflow2'])
]
can_edit = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator':
coordinator.get_data_for_json(),
'credentials':
list(credentials.credentials.keys()),
'workflows':
workflows,
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit':
can_edit,
'layout':
django_mako.render_to_string(
'editor2/common_scheduler.mako',
{'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render(
'editor2/coordinator_editor.mako', request, {
'coordinator_json':
coordinator.to_json_for_html(),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflows_json':
json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_json':
json.dumps(can_edit)
})
