def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def submit_coordinator(request, doc_id):
coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
job_id = _submit_coordinator(request, coordinator, mapping)
request.info(_('Coordinator submitted.'))
return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = coordinator.find_all_parameters()
initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor2/submit_job_popup.mako', request, {
'params_form': params_form,
'name': coordinator.name,
'action': reverse('oozie:editor_submit_coordinator', kwargs={'doc_id': coordinator.id}),
'show_dryrun': True
}, force_template=True).content
return JsonResponse(popup, safe=False)
def copy_coordinator(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
doc2 = Document2.objects.get(type='oozie-coordinator2', id=job['id'])
doc = doc2.doc.get()
name = doc2.name + '-copy'
doc2 = doc2.copy(name=name, owner=request.user)
doc.copy(content_object=doc2, name=name, owner=request.user)
coord = Coordinator(document=doc2)
coordinator_data = coord.get_data_for_json()
coordinator_data['name'] = name
_import_workspace(request.fs, request.user, coord)
doc2.update_data(coordinator_data)
doc2.save()
response = {}
request.info(_('Coordinator copied.') if len(jobs) > 1 else _('Coordinator copied.'))
return JsonResponse(response)
def copy_coordinator(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
doc2 = Document2.objects.get(type='oozie-coordinator2', id=job['id'])
doc = doc2.doc.get()
name = doc2.name + '-copy'
doc2 = doc2.copy(name=name, owner=request.user)
doc.copy(content_object=doc2, name=name, owner=request.user)
coord = Coordinator(document=doc2)
coordinator_data = coord.get_data_for_json()
coordinator_data['name'] = name
_import_workspace(request.fs, request.user, coord)
doc2.update_data(coordinator_data)
doc2.save()
response = {}
request.info(
_('Coordinator copied.') if len(jobs) > 1 else _('Coordinator copied.')
)
return JsonResponse(response)
def submit_coordinator(request, doc_id):
coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
jsonify = request.POST.get('format') == 'json'
job_id = _submit_coordinator(request, coordinator, mapping)
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
else:
request.info(_('Coordinator submitted.'))
return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = coordinator.find_all_parameters()
initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor2/submit_job_popup.mako', request, {
'params_form': params_form,
'name': coordinator.name,
'action': reverse('oozie:editor_submit_coordinator', kwargs={'doc_id': coordinator.id}),
'show_dryrun': True,
'return_json': request.GET.get('format') == 'json'
}, force_template=True).content
return JsonResponse(popup, safe=False)
def coordinator_parameters(request):
response = {'status': -1}
try:
coordinator = Coordinator(document=Document2.objects.get(type='oozie-coordinator2', uuid=request.GET.get('uuid')))
response['status'] = 0
response['parameters'] = coordinator.find_all_parameters(with_lib_path=False)
except Exception, e:
response['message'] = str(e)
def coordinator_parameters(request):
response = {'status': -1}
try:
coordinator = Coordinator(document=Document2.objects.get(type='oozie-coordinator2', uuid=request.GET.get('uuid')))
response['status'] = 0
response['parameters'] = coordinator.find_all_parameters(with_lib_path=False)
except Exception, e:
response['message'] = str(e)
def gen_xml_coordinator(request):
response = {'status': -1}
coordinator_dict = json.loads(request.POST.get('coordinator', '{}'))
coordinator = Coordinator(data=coordinator_dict)
response['status'] = 0
response['xml'] = coordinator.to_xml()
return JsonResponse(response)
def gen_xml_coordinator(request):
response = {'status': -1}
coordinator_dict = json.loads(request.POST.get('coordinator', '{}'))
coordinator = Coordinator(data=coordinator_dict)
response['status'] = 0
response['xml'] = coordinator.to_xml()
return HttpResponse(json.dumps(response), mimetype="application/json")
def gen_xml_coordinator(request):
response = {'status': -1}
coordinator_dict = json.loads(request.POST.get('coordinator', '{}'))
coordinator = Coordinator(data=coordinator_dict)
response['status'] = 0
response['xml'] = coordinator.to_xml()
return JsonResponse(response)
def copy_coordinator(request):
if request.method != 'POST':
raise PopupException(_('A POST request is required.'))
jobs = json.loads(request.POST.get('selection'))
for job in jobs:
doc2 = Document2.objects.get(type='oozie-coordinator2', id=job['id'])
name = doc2.name + '-copy'
copy_doc = doc2.doc.get().copy(name=name, owner=request.user)
doc2.pk = None
doc2.id = None
doc2.uuid = str(uuid.uuid4())
doc2.name = name
doc2.owner = request.user
doc2.save()
doc2.doc.all().delete()
doc2.doc.add(copy_doc)
coordinator_data = Coordinator(document=doc2).get_data_for_json()
coordinator_data['name'] = name
doc2.update_data(coordinator_data)
doc2.save()
response = {}
request.info(_('Coordinator copied.') if len(jobs) > 1 else _('Coordinator copied.'))
return JsonResponse(response)
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy()
deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir)
deployment_mapping['coord_%s' % i] = coord
# Convert start/end dates of coordinator to server timezone
for prop in bundled['properties']:
if prop['name'] in ('end_date', 'start_date'):
prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone'])
properties.update(deployment_mapping)
submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
LOG.exception('Error submitting bundle')
raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def _submit_bundle(request, bundle, properties):
try:
deployment_mapping = {}
coords = dict([(c.uuid, c) for c in Document2.objects.filter(
type='oozie-coordinator2',
uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])])
for i, bundled in enumerate(bundle.data['coordinators']):
coord = coords[bundled['coordinator']]
workflow = Workflow(document=coord.dependencies.all()[0])
wf_dir = Submission(request.user, workflow, request.fs, request.jt,
properties).deploy()
deployment_mapping['wf_%s_dir' %
i] = request.fs.get_hdfs_path(wf_dir)
coordinator = Coordinator(document=coord)
coord_dir = Submission(request.user, coordinator, request.fs,
request.jt, properties).deploy()
deployment_mapping['coord_%s_dir' % i] = coord_dir
deployment_mapping['coord_%s' % i] = coord
properties.update(deployment_mapping)
submission = Submission(request.user,
bundle,
request.fs,
request.jt,
properties=properties)
job_id = submission.run()
return job_id
except RestException, ex:
raise PopupException(_("Error submitting bundle %s") % (bundle, ),
detail=ex._headers.get('oozie-error-message', ex))
def copy_document(request):
uuid = json.loads(request.POST.get('uuid'), '""')
if not uuid:
raise PopupException(_('copy_document requires uuid'))
document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid)
if document.type == 'directory':
raise PopupException(_('Directory copy is not supported'))
name = document.name + '-copy'
# Make the copy of the new Document
copy_document = document.copy(name=name, owner=request.user)
# Import workspace for all oozie jobs
if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace
# Update the name field in the json 'data' field
if document.type == 'oozie-workflow2':
workflow = Workflow(document=document)
workflow.update_name(name)
workflow.update_uuid(copy_document.uuid)
_import_workspace(request.fs, request.user, workflow)
copy_document.update_data(
{'workflow': workflow.get_data()['workflow']})
copy_document.save()
if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2':
if document.type == 'oozie-bundle2':
bundle_or_coordinator = Bundle(document=document)
else:
bundle_or_coordinator = Coordinator(document=document)
json_data = bundle_or_coordinator.get_data_for_json()
json_data['name'] = name
json_data['uuid'] = copy_document.uuid
copy_document.update_data(json_data)
copy_document.save()
_import_workspace(request.fs, request.user, bundle_or_coordinator)
elif document.type == 'search-dashboard':
from dashboard.models import Collection2
collection = Collection2(request.user, document=document)
collection.data['collection']['label'] = name
collection.data['collection']['uuid'] = copy_document.uuid
copy_document.update_data(
{'collection': collection.data['collection']})
copy_document.save()
# Keep the document and data in sync
else:
copy_data = copy_document.data_dict
if 'name' in copy_data:
copy_data['name'] = name
if 'uuid' in copy_data:
copy_data['uuid'] = copy_document.uuid
copy_document.update_data(copy_data)
copy_document.save()
return JsonResponse({'status': 0, 'document': copy_document.to_dict()})
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def submit_coordinator(request, doc_id):
# TODO: Replace URL by desktop/scheduler API
if doc_id.isdigit():
coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
else:
coordinator = Coordinator(document=Document2.objects.get_by_uuid(
user=request.user, uuid=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value'])
for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
jsonify = request.POST.get('format') == 'json'
try:
job_id = _submit_coordinator(request, coordinator, mapping)
except Exception, e:
message = force_unicode(str(e))
return JsonResponse({
'status': -1,
'message': message
},
safe=False)
if jsonify:
return JsonResponse(
{
'status': 0,
'job_id': job_id,
'type': 'schedule'
},
safe=False)
else:
request.info(_('Coordinator submitted.'))
return redirect(
reverse('oozie:list_oozie_coordinator',
kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' %
params_form.errors))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(
type='oozie-workflow2',
owner=request.user,
is_managed=True,
dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user,
uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document,
user=request.user,
managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def submit_coordinator(request, doc_id):
# TODO: Replace URL by desktop/scheduler API
if doc_id.isdigit():
coordinator = Coordinator(document=Document2.objects.get(id=doc_id))
else:
coordinator = Coordinator(document=Document2.objects.get_by_uuid(user=request.user, uuid=doc_id))
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
jsonify = request.POST.get('format') == 'json'
try:
job_id = _submit_coordinator(request, coordinator, mapping)
except Exception as e:
message = force_unicode(str(e))
return JsonResponse({'status': -1, 'message': message}, safe=False)
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'schedule'}, safe=False)
else:
request.info(_('Coordinator submitted.'))
return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = coordinator.find_all_parameters()
initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters]))
params_form = ParametersFormSet(initial=initial_params)
return render('/scheduler/submit_job_popup.mako', request, {
'params_form': params_form,
'name': coordinator.name,
'action': reverse('oozie:editor_submit_coordinator', kwargs={'doc_id': coordinator.id}),
'show_dryrun': True,
'return_json': request.GET.get('format') == 'json'
}, force_template=True)
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
# Automatically create the workflow of a scheduled document
# To move to save coordinator
document_uuid = request.GET.get('document')
if document_uuid:
# Has already a workflow managing the query for this user?
workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid])
if workflows.exists():
workflow_doc = workflows.get()
else:
document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid)
workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True)
if doc:
doc.dependencies.add(workflow_doc)
workflow_uuid = workflow_doc.uuid
coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name
elif request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator')
doc = None
if coordinator_id:
doc = Document2.objects.get(id=coordinator_id)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception, e:
LOG.error(smart_str(e))
def edit_coordinator(request):
coordinator_id = request.GET.get('coordinator', request.GET.get('uuid'))
doc = None
workflow_uuid = None
if coordinator_id:
cid = {}
if coordinator_id.isdigit():
cid['id'] = coordinator_id
else:
cid['uuid'] = coordinator_id
doc = Document2.objects.get(**cid)
coordinator = Coordinator(document=doc)
else:
coordinator = Coordinator()
coordinator.set_workspace(request.user)
if request.GET.get('workflow'):
workflow_uuid = request.GET.get('workflow')
if workflow_uuid:
coordinator.data['properties']['workflow'] = workflow_uuid
api = get_oozie(request.user)
credentials = Credentials()
try:
credentials.fetch(api)
except Exception as e:
LOG.error(smart_str(e))
if USE_NEW_EDITOR.get():
scheduled_uuid = coordinator.data['properties'][
'workflow'] or coordinator.data['properties']['document']
if scheduled_uuid:
try:
document = Document2.objects.get(uuid=scheduled_uuid)
except Document2.DoesNotExist as e:
document = None
coordinator.data['properties']['workflow'] = ''
LOG.warning("Workflow with uuid %s doesn't exist: %s" %
(scheduled_uuid, e))
if document and document.is_trashed:
raise PopupException(
_('Your workflow %s has been trashed!') %
(document.name if document.name else ''))
if document and not document.can_read(request.user):
raise PopupException(
_('You don\'t have access to the workflow or document of this coordinator.'
))
else:
workflows = [
dict([('uuid', d.content_object.uuid),
('name', d.content_object.name)])
for d in Document.objects.available_docs(
Document2, request.user).filter(extra='workflow2')
]
if coordinator_id and not [
a for a in workflows
if a['uuid'] == coordinator.data['properties']['workflow']
]:
raise PopupException(
_('You don\'t have access to the workflow of this coordinator.'
))
if USE_NEW_EDITOR.get(): # In Hue 4, merge with above
workflows = [
dict([('uuid', d.uuid), ('name', d.name)])
for d in Document2.objects.documents(
request.user, include_managed=False).search_documents(
types=['oozie-workflow2'])
]
can_edit = doc is None or (doc.can_write(request.user)
if USE_NEW_EDITOR.get() else
doc.doc.get().is_editable(request.user))
if request.GET.get('format') == 'json': # For Editor
return JsonResponse({
'coordinator':
coordinator.get_data_for_json(),
'credentials':
list(credentials.credentials.keys()),
'workflows':
workflows,
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit':
can_edit,
'layout':
django_mako.render_to_string(
'editor2/common_scheduler.mako',
{'coordinator_json': coordinator.to_json_for_html()})
})
else:
return render(
'editor2/coordinator_editor.mako', request, {
'coordinator_json':
coordinator.to_json_for_html(),
'credentials_json':
json.dumps(list(credentials.credentials.keys()),
cls=JSONEncoderForHTML),
'workflows_json':
json.dumps(workflows, cls=JSONEncoderForHTML),
'doc_uuid':
doc.uuid if doc else '',
'is_embeddable':
request.GET.get('is_embeddable', False),
'can_edit_json':
json.dumps(can_edit)
})
