def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) if request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def submit_coordinator(request, doc_id): coordinator = Coordinator(document=Document2.objects.get(id=doc_id)) ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' job_id = _submit_coordinator(request, coordinator, mapping) request.info(_('Coordinator submitted.')) return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors)) else: parameters = coordinator.find_all_parameters() initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters])) params_form = ParametersFormSet(initial=initial_params) popup = render('editor2/submit_job_popup.mako', request, { 'params_form': params_form, 'name': coordinator.name, 'action': reverse('oozie:editor_submit_coordinator', kwargs={'doc_id': coordinator.id}), 'show_dryrun': True }, force_template=True).content return JsonResponse(popup, safe=False)
def copy_coordinator(request): if request.method != 'POST': raise PopupException(_('A POST request is required.')) jobs = json.loads(request.POST.get('selection')) for job in jobs: doc2 = Document2.objects.get(type='oozie-coordinator2', id=job['id']) doc = doc2.doc.get() name = doc2.name + '-copy' doc2 = doc2.copy(name=name, owner=request.user) doc.copy(content_object=doc2, name=name, owner=request.user) coord = Coordinator(document=doc2) coordinator_data = coord.get_data_for_json() coordinator_data['name'] = name _import_workspace(request.fs, request.user, coord) doc2.update_data(coordinator_data) doc2.save() response = {} request.info(_('Coordinator copied.') if len(jobs) > 1 else _('Coordinator copied.')) return JsonResponse(response)
def copy_coordinator(request): if request.method != 'POST': raise PopupException(_('A POST request is required.')) jobs = json.loads(request.POST.get('selection')) for job in jobs: doc2 = Document2.objects.get(type='oozie-coordinator2', id=job['id']) doc = doc2.doc.get() name = doc2.name + '-copy' doc2 = doc2.copy(name=name, owner=request.user) doc.copy(content_object=doc2, name=name, owner=request.user) coord = Coordinator(document=doc2) coordinator_data = coord.get_data_for_json() coordinator_data['name'] = name _import_workspace(request.fs, request.user, coord) doc2.update_data(coordinator_data) doc2.save() response = {} request.info( _('Coordinator copied.') if len(jobs) > 1 else _('Coordinator copied.') ) return JsonResponse(response)
def submit_coordinator(request, doc_id): coordinator = Coordinator(document=Document2.objects.get(id=doc_id)) ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' jsonify = request.POST.get('format') == 'json' job_id = _submit_coordinator(request, coordinator, mapping) if jsonify: return JsonResponse({'status': 0, 'job_id': job_id}, safe=False) else: request.info(_('Coordinator submitted.')) return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors)) else: parameters = coordinator.find_all_parameters() initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters])) params_form = ParametersFormSet(initial=initial_params) popup = render('editor2/submit_job_popup.mako', request, { 'params_form': params_form, 'name': coordinator.name, 'action': reverse('oozie:editor_submit_coordinator', kwargs={'doc_id': coordinator.id}), 'show_dryrun': True, 'return_json': request.GET.get('format') == 'json' }, force_template=True).content return JsonResponse(popup, safe=False)
def coordinator_parameters(request): response = {'status': -1} try: coordinator = Coordinator(document=Document2.objects.get(type='oozie-coordinator2', uuid=request.GET.get('uuid'))) response['status'] = 0 response['parameters'] = coordinator.find_all_parameters(with_lib_path=False) except Exception, e: response['message'] = str(e)
def gen_xml_coordinator(request): response = {'status': -1} coordinator_dict = json.loads(request.POST.get('coordinator', '{}')) coordinator = Coordinator(data=coordinator_dict) response['status'] = 0 response['xml'] = coordinator.to_xml() return JsonResponse(response)
def gen_xml_coordinator(request): response = {'status': -1} coordinator_dict = json.loads(request.POST.get('coordinator', '{}')) coordinator = Coordinator(data=coordinator_dict) response['status'] = 0 response['xml'] = coordinator.to_xml() return HttpResponse(json.dumps(response), mimetype="application/json")
def copy_coordinator(request): if request.method != 'POST': raise PopupException(_('A POST request is required.')) jobs = json.loads(request.POST.get('selection')) for job in jobs: doc2 = Document2.objects.get(type='oozie-coordinator2', id=job['id']) name = doc2.name + '-copy' copy_doc = doc2.doc.get().copy(name=name, owner=request.user) doc2.pk = None doc2.id = None doc2.uuid = str(uuid.uuid4()) doc2.name = name doc2.owner = request.user doc2.save() doc2.doc.all().delete() doc2.doc.add(copy_doc) coordinator_data = Coordinator(document=doc2).get_data_for_json() coordinator_data['name'] = name doc2.update_data(coordinator_data) doc2.save() response = {} request.info(_('Coordinator copied.') if len(jobs) > 1 else _('Coordinator copied.')) return JsonResponse(response)
def _submit_bundle(request, bundle, properties): try: deployment_mapping = {} coords = dict([(c.uuid, c) for c in Document2.objects.filter(type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])]) for i, bundled in enumerate(bundle.data['coordinators']): coord = coords[bundled['coordinator']] workflow = Workflow(document=coord.dependencies.filter(type='oozie-workflow2')[0]) wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy() deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir) coordinator = Coordinator(document=coord) coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy() deployment_mapping['coord_%s_dir' % i] = request.fs.get_hdfs_path(coord_dir) deployment_mapping['coord_%s' % i] = coord # Convert start/end dates of coordinator to server timezone for prop in bundled['properties']: if prop['name'] in ('end_date', 'start_date'): prop['value'] = convert_to_server_timezone(prop['value'], local_tz=coordinator.data['properties']['timezone']) properties.update(deployment_mapping) submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties) job_id = submission.run() return job_id except RestException, ex: LOG.exception('Error submitting bundle') raise PopupException(_("Error submitting bundle %s") % (bundle,), detail=ex._headers.get('oozie-error-message', ex))
def _submit_bundle(request, bundle, properties): try: deployment_mapping = {} coords = dict([(c.uuid, c) for c in Document2.objects.filter( type='oozie-coordinator2', uuid__in=[b['coordinator'] for b in bundle.data['coordinators']])]) for i, bundled in enumerate(bundle.data['coordinators']): coord = coords[bundled['coordinator']] workflow = Workflow(document=coord.dependencies.all()[0]) wf_dir = Submission(request.user, workflow, request.fs, request.jt, properties).deploy() deployment_mapping['wf_%s_dir' % i] = request.fs.get_hdfs_path(wf_dir) coordinator = Coordinator(document=coord) coord_dir = Submission(request.user, coordinator, request.fs, request.jt, properties).deploy() deployment_mapping['coord_%s_dir' % i] = coord_dir deployment_mapping['coord_%s' % i] = coord properties.update(deployment_mapping) submission = Submission(request.user, bundle, request.fs, request.jt, properties=properties) job_id = submission.run() return job_id except RestException, ex: raise PopupException(_("Error submitting bundle %s") % (bundle, ), detail=ex._headers.get('oozie-error-message', ex))
def copy_document(request): uuid = json.loads(request.POST.get('uuid'), '""') if not uuid: raise PopupException(_('copy_document requires uuid')) document = Document2.objects.get_by_uuid(user=request.user, uuid=uuid) if document.type == 'directory': raise PopupException(_('Directory copy is not supported')) name = document.name + '-copy' # Make the copy of the new Document copy_document = document.copy(name=name, owner=request.user) # Import workspace for all oozie jobs if document.type == 'oozie-workflow2' or document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2': from oozie.models2 import Workflow, Coordinator, Bundle, _import_workspace # Update the name field in the json 'data' field if document.type == 'oozie-workflow2': workflow = Workflow(document=document) workflow.update_name(name) workflow.update_uuid(copy_document.uuid) _import_workspace(request.fs, request.user, workflow) copy_document.update_data( {'workflow': workflow.get_data()['workflow']}) copy_document.save() if document.type == 'oozie-bundle2' or document.type == 'oozie-coordinator2': if document.type == 'oozie-bundle2': bundle_or_coordinator = Bundle(document=document) else: bundle_or_coordinator = Coordinator(document=document) json_data = bundle_or_coordinator.get_data_for_json() json_data['name'] = name json_data['uuid'] = copy_document.uuid copy_document.update_data(json_data) copy_document.save() _import_workspace(request.fs, request.user, bundle_or_coordinator) elif document.type == 'search-dashboard': from dashboard.models import Collection2 collection = Collection2(request.user, document=document) collection.data['collection']['label'] = name collection.data['collection']['uuid'] = copy_document.uuid copy_document.update_data( {'collection': collection.data['collection']}) copy_document.save() # Keep the document and data in sync else: copy_data = copy_document.data_dict if 'name' in copy_data: copy_data['name'] = name if 'uuid' in copy_data: copy_data['uuid'] = copy_document.uuid copy_document.update_data(copy_data) copy_document.save() return JsonResponse({'status': 0, 'document': copy_document.to_dict()})
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def submit_coordinator(request, doc_id): # TODO: Replace URL by desktop/scheduler API if doc_id.isdigit(): coordinator = Coordinator(document=Document2.objects.get(id=doc_id)) else: coordinator = Coordinator(document=Document2.objects.get_by_uuid( user=request.user, uuid=doc_id)) ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' jsonify = request.POST.get('format') == 'json' try: job_id = _submit_coordinator(request, coordinator, mapping) except Exception, e: message = force_unicode(str(e)) return JsonResponse({ 'status': -1, 'message': message }, safe=False) if jsonify: return JsonResponse( { 'status': 0, 'job_id': job_id, 'type': 'schedule' }, safe=False) else: request.info(_('Coordinator submitted.')) return redirect( reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors))
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) # Automatically create the workflow of a scheduled document # To move to save coordinator document_uuid = request.GET.get('document') if document_uuid: # Has already a workflow managing the query for this user? workflows = Document2.objects.filter( type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid]) if workflows.exists(): workflow_doc = workflows.get() else: document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid) workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True) if doc: doc.dependencies.add(workflow_doc) workflow_uuid = workflow_doc.uuid coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name elif request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def submit_coordinator(request, doc_id): # TODO: Replace URL by desktop/scheduler API if doc_id.isdigit(): coordinator = Coordinator(document=Document2.objects.get(id=doc_id)) else: coordinator = Coordinator(document=Document2.objects.get_by_uuid(user=request.user, uuid=doc_id)) ParametersFormSet = formset_factory(ParameterForm, extra=0) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' jsonify = request.POST.get('format') == 'json' try: job_id = _submit_coordinator(request, coordinator, mapping) except Exception as e: message = force_unicode(str(e)) return JsonResponse({'status': -1, 'message': message}, safe=False) if jsonify: return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'schedule'}, safe=False) else: request.info(_('Coordinator submitted.')) return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors)) else: parameters = coordinator.find_all_parameters() initial_params = ParameterForm.get_initial_params(dict([(param['name'], param['value']) for param in parameters])) params_form = ParametersFormSet(initial=initial_params) return render('/scheduler/submit_job_popup.mako', request, { 'params_form': params_form, 'name': coordinator.name, 'action': reverse('oozie:editor_submit_coordinator', kwargs={'doc_id': coordinator.id}), 'show_dryrun': True, 'return_json': request.GET.get('format') == 'json' }, force_template=True)
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) # Automatically create the workflow of a scheduled document # To move to save coordinator document_uuid = request.GET.get('document') if document_uuid: # Has already a workflow managing the query for this user? workflows = Document2.objects.filter(type='oozie-workflow2', owner=request.user, is_managed=True, dependencies__uuid__in=[document_uuid]) if workflows.exists(): workflow_doc = workflows.get() else: document = Document2.objects.get_by_uuid(user=request.user, uuid=document_uuid) workflow_doc = WorkflowBuilder().create_workflow(document=document, user=request.user, managed=True) if doc: doc.dependencies.add(workflow_doc) workflow_uuid = workflow_doc.uuid coordinator.data['name'] = _('Schedule of %s') % workflow_doc.name elif request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator') doc = None if coordinator_id: doc = Document2.objects.get(id=coordinator_id) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception, e: LOG.error(smart_str(e))
def edit_coordinator(request): coordinator_id = request.GET.get('coordinator', request.GET.get('uuid')) doc = None workflow_uuid = None if coordinator_id: cid = {} if coordinator_id.isdigit(): cid['id'] = coordinator_id else: cid['uuid'] = coordinator_id doc = Document2.objects.get(**cid) coordinator = Coordinator(document=doc) else: coordinator = Coordinator() coordinator.set_workspace(request.user) if request.GET.get('workflow'): workflow_uuid = request.GET.get('workflow') if workflow_uuid: coordinator.data['properties']['workflow'] = workflow_uuid api = get_oozie(request.user) credentials = Credentials() try: credentials.fetch(api) except Exception as e: LOG.error(smart_str(e)) if USE_NEW_EDITOR.get(): scheduled_uuid = coordinator.data['properties'][ 'workflow'] or coordinator.data['properties']['document'] if scheduled_uuid: try: document = Document2.objects.get(uuid=scheduled_uuid) except Document2.DoesNotExist as e: document = None coordinator.data['properties']['workflow'] = '' LOG.warning("Workflow with uuid %s doesn't exist: %s" % (scheduled_uuid, e)) if document and document.is_trashed: raise PopupException( _('Your workflow %s has been trashed!') % (document.name if document.name else '')) if document and not document.can_read(request.user): raise PopupException( _('You don\'t have access to the workflow or document of this coordinator.' )) else: workflows = [ dict([('uuid', d.content_object.uuid), ('name', d.content_object.name)]) for d in Document.objects.available_docs( Document2, request.user).filter(extra='workflow2') ] if coordinator_id and not [ a for a in workflows if a['uuid'] == coordinator.data['properties']['workflow'] ]: raise PopupException( _('You don\'t have access to the workflow of this coordinator.' )) if USE_NEW_EDITOR.get(): # In Hue 4, merge with above workflows = [ dict([('uuid', d.uuid), ('name', d.name)]) for d in Document2.objects.documents( request.user, include_managed=False).search_documents( types=['oozie-workflow2']) ] can_edit = doc is None or (doc.can_write(request.user) if USE_NEW_EDITOR.get() else doc.doc.get().is_editable(request.user)) if request.GET.get('format') == 'json': # For Editor return JsonResponse({ 'coordinator': coordinator.get_data_for_json(), 'credentials': list(credentials.credentials.keys()), 'workflows': workflows, 'doc_uuid': doc.uuid if doc else '', 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit': can_edit, 'layout': django_mako.render_to_string( 'editor2/common_scheduler.mako', {'coordinator_json': coordinator.to_json_for_html()}) }) else: return render( 'editor2/coordinator_editor.mako', request, { 'coordinator_json': coordinator.to_json_for_html(), 'credentials_json': json.dumps(list(credentials.credentials.keys()), cls=JSONEncoderForHTML), 'workflows_json': json.dumps(workflows, cls=JSONEncoderForHTML), 'doc_uuid': doc.uuid if doc else '', 'is_embeddable': request.GET.get('is_embeddable', False), 'can_edit_json': json.dumps(can_edit) })