def continue_request(self, data, request): logger = self.supervisord.options.logger try: params, method = self.loads(data) # no <methodName> in the request or name is an empty string if not method: logger.trace('XML-RPC request received with no method name') request.error(400) return # we allow xml-rpc clients that do not send empty <params> # when there are no parameters for the method call if params is None: params = () try: logger.trace('XML-RPC method called: %s()' % method) value = self.call(method, params) # application-specific: instead of we never want to # marshal None (even though we could by saying allow_none=True # in dumps within xmlrpc_marshall), this is meant as # a debugging fixture, see issue 223. assert value is not None, ( 'return value from method %r with params %r is None' % (method, params)) logger.trace('XML-RPC method %s() returned successfully' % method) except RPCError as err: # turn RPCError reported by method into a Fault instance value = xmlrpc.client.Fault(err.code, err.text) logger.trace('XML-RPC method %s() returned fault: [%d] %s' % (method, err.code, err.text)) if isinstance(value, types.FunctionType): # returning a function from an RPC method implies that # this needs to be a deferred response (it needs to block). pushproducer = request.channel.push_with_producer pushproducer(DeferredXMLRPCResponse(request, value)) else: # if we get anything but a function, it implies that this # response doesn't need to be deferred, we can service it # right away. body = xmlrpc_marshal(value) request['Content-Type'] = 'text/xml' request['Content-Length'] = len(body) request.push(body) request.done() except: io = StringIO() traceback.print_exc(file=io) val = io.getvalue() logger.critical(val) # internal error, report as HTTP server error request.error(500)
def submit_external_job(request, application_path): ParametersFormSet = formset_factory(ParameterForm, extra=0) if application_path.startswith('abfs:/') and not application_path.startswith('abfs://'): application_path = application_path.replace("abfs:/", "abfs://") elif application_path.startswith('s3a:/') and not application_path.startswith('s3a://'): application_path = application_path.replace('s3a:/', 's3a://') else: application_path = "/" + application_path if application_path.startswith("abfs://"): application_path = abfspath(application_path) if request.method == 'POST': params_form = ParametersFormSet(request.POST) if params_form.is_valid(): mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on' application_name = os.path.basename(application_path) application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow() mapping[application_class.get_application_path_key()] = os.path.dirname(application_path) try: submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping) job_id = submission.run(application_path) except RestException as ex: detail = ex._headers.get('oozie-error-message', ex) if 'Max retries exceeded with url' in str(detail): detail = '%s: %s' % (_('The Oozie server is not running'), detail) LOG.exception(smart_str(detail)) raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail) jsonify = request.POST.get('format') == 'json' if jsonify: return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False) else: request.info(_('Oozie job submitted')) view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow' return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % params_form.errors)) else: parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path) initial_params = ParameterForm.get_initial_params(parameters) params_form = ParametersFormSet(initial=initial_params) popup = render('editor/submit_job_popup.mako', request, { 'params_form': params_form, 'name': _('Job'), 'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}), 'show_dryrun': os.path.basename(application_path) != 'bundle.xml', 'return_json': request.GET.get('format') == 'json' }, force_template=True).content return JsonResponse(popup, safe=False)
def rerun_oozie_coordinator(request, job_id, app_path=None): oozie_coordinator = check_job_access_permission(request, job_id) check_job_edition_permission(oozie_coordinator, request.user) ParametersFormSet = formset_factory(ParameterForm, extra=0) if app_path is None: app_path = oozie_coordinator.coordJobPath else: app_path = urllib.parse.unquote(app_path) return_json = request.GET.get('format') == 'json' if request.method == 'POST': params_form = ParametersFormSet(request.POST) rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator) if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2: args = {} args['deployment_dir'] = app_path params = { 'type': 'action', 'scope': ','.join(oozie_coordinator.aggreate(rerun_form.cleaned_data['actions'])), 'refresh': rerun_form.cleaned_data['refresh'], 'nocleanup': rerun_form.cleaned_data['nocleanup'], } properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) _rerun_coordinator(request, job_id, args, params, properties) if rerun_form.cleaned_data['return_json']: return JsonResponse({'status': 0, 'job_id': job_id}, safe=False) else: request.info(_('Coordinator re-running.')) return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s') % smart_unicode(rerun_form.errors)) return list_oozie_coordinator(request, job_id) else: rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator, return_json=return_json) initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict) params_form = ParametersFormSet(initial=initial_params) return render('dashboard/rerun_coord_popup.mako', request, { 'rerun_form': rerun_form, 'params_form': params_form, 'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}), 'return_json': return_json, 'is_mini': request.GET.get('is_mini', False), }, force_template=True)
def rerun_oozie_bundle(request, job_id, app_path): oozie_bundle = check_job_access_permission(request, job_id) check_job_edition_permission(oozie_bundle, request.user) ParametersFormSet = formset_factory(ParameterForm, extra=0) app_path = urllib.parse.unquote(app_path) if request.method == 'POST': params_form = ParametersFormSet(request.POST) rerun_form = RerunBundleForm(request.POST, oozie_bundle=oozie_bundle) if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2: args = {} args['deployment_dir'] = app_path params = { 'coord-scope': ','.join(rerun_form.cleaned_data['coordinators']), 'refresh': rerun_form.cleaned_data['refresh'], 'nocleanup': rerun_form.cleaned_data['nocleanup'], } if rerun_form.cleaned_data['start'] and rerun_form.cleaned_data['end']: date = { 'date-scope': '%(start)s::%(end)s' % { 'start': utc_datetime_format(rerun_form.cleaned_data['start']), 'end': utc_datetime_format(rerun_form.cleaned_data['end']) } } params.update(date) properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) _rerun_bundle(request, job_id, args, params, properties) request.info(_('Bundle re-running.')) return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s' % (rerun_form.errors,))) return list_oozie_bundle(request, job_id) else: rerun_form = RerunBundleForm(oozie_bundle=oozie_bundle) initial_params = ParameterForm.get_initial_params(oozie_bundle.conf_dict) params_form = ParametersFormSet(initial=initial_params) return render('dashboard/rerun_bundle_popup.mako', request, { 'rerun_form': rerun_form, 'params_form': params_form, 'action': reverse('oozie:rerun_oozie_bundle', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}), }, force_template=True)
def rerun_oozie_job(request, job_id, app_path=None): ParametersFormSet = formset_factory(ParameterForm, extra=0) oozie_workflow = check_job_access_permission(request, job_id) check_job_edition_permission(oozie_workflow, request.user) if app_path is None: app_path = oozie_workflow.appPath else: app_path = urllib.parse.unquote(app_path) return_json = request.GET.get('format') == 'json' if request.method == 'POST': rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow) params_form = ParametersFormSet(request.POST) if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2: args = {} if request.POST.get('rerun_form_choice') == 'fail_nodes': args['fail_nodes'] = 'true' else: args['skip_nodes'] = ','.join(rerun_form.cleaned_data['skip_nodes']) args['deployment_dir'] = app_path mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data]) _rerun_workflow(request, job_id, args, mapping) if rerun_form.cleaned_data['return_json']: return JsonResponse({'status': 0, 'job_id': job_id}, safe=False) else: request.info(_('Workflow re-running.')) return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id})) else: request.error(_('Invalid submission form: %s %s' % (rerun_form.errors, params_form.errors))) else: rerun_form = RerunForm(oozie_workflow=oozie_workflow, return_json=return_json) initial_params = ParameterForm.get_initial_params(oozie_workflow.conf_dict) params_form = ParametersFormSet(initial=initial_params) return render('dashboard/rerun_workflow_popup.mako', request, { 'rerun_form': rerun_form, 'params_form': params_form, 'action': reverse('oozie:rerun_oozie_job', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) }), 'return_json': return_json, 'is_mini': request.GET.get('is_mini', False), }, force_template=True)
def handle_request(self, request): if request.command != 'GET': request.error (400) # bad request return path, params, query, fragment = request.split_uri() if '%' in path: path = http_server.unquote(path) # strip off all leading slashes while path and path[0] == '/': path = path[1:] path, process_name_and_channel = path.split('/', 1) try: process_name, channel = process_name_and_channel.split('/', 1) except ValueError: # no channel specified, default channel to stdout process_name = process_name_and_channel channel = 'stdout' from supervisor.options import split_namespec group_name, process_name = split_namespec(process_name) group = self.supervisord.process_groups.get(group_name) if group is None: request.error(404) # not found return process = group.processes.get(process_name) if process is None: request.error(404) # not found return logfile = getattr(process.config, '%s_logfile' % channel, None) if logfile is None or not os.path.exists(logfile): # XXX problematic: processes that don't start won't have a log # file and we probably don't want to go into fatal state if we try # to read the log of a process that did not start. request.error(410) # gone return mtime = os.stat(logfile)[stat.ST_MTIME] request['Last-Modified'] = http_date.build_http_date(mtime) request['Content-Type'] = 'text/plain' # the lack of a Content-Length header makes the outputter # send a 'Transfer-Encoding: chunked' response request.push(tail_f_producer(request, logfile, 1024)) request.done()
def handle_request(self, request): if request.command != 'GET': request.error(400) # bad request return path, params, query, fragment = request.split_uri() if '%' in path: path = http_server.unquote(path) # strip off all leading slashes while path and path[0] == '/': path = path[1:] path, process_name_and_channel = path.split('/', 1) try: process_name, channel = process_name_and_channel.split('/', 1) except ValueError: # no channel specified, default channel to stdout process_name = process_name_and_channel channel = 'stdout' from supervisor.options import split_namespec group_name, process_name = split_namespec(process_name) group = self.supervisord.process_groups.get(group_name) if group is None: request.error(404) # not found return process = group.processes.get(process_name) if process is None: request.error(404) # not found return logfile = getattr(process.config, '%s_logfile' % channel, None) if logfile is None or not os.path.exists(logfile): # XXX problematic: processes that don't start won't have a log # file and we probably don't want to go into fatal state if we try # to read the log of a process that did not start. request.error(410) # gone return mtime = os.stat(logfile)[stat.ST_MTIME] request['Last-Modified'] = http_date.build_http_date(mtime) request['Content-Type'] = 'text/plain' # the lack of a Content-Length header makes the outputter # send a 'Transfer-Encoding: chunked' response request.push(tail_f_producer(request, logfile, 1024)) request.done()
def handle_request(self, request): if request.command != 'GET': request.error (400) # bad request return logfile = self.supervisord.options.logfile if logfile is None or not os.path.exists(logfile): request.error(410) # gone return mtime = os.stat(logfile)[stat.ST_MTIME] request['Last-Modified'] = http_date.build_http_date(mtime) request['Content-Type'] = 'text/plain' # the lack of a Content-Length header makes the outputter # send a 'Transfer-Encoding: chunked' response request.push(tail_f_producer(request, logfile, 1024)) request.done()
def handle_request(self, request): if request.command != 'GET': request.error(400) # bad request return logfile = self.supervisord.options.logfile if logfile is None or not os.path.exists(logfile): request.error(410) # gone return mtime = os.stat(logfile)[stat.ST_MTIME] request['Last-Modified'] = http_date.build_http_date(mtime) request['Content-Type'] = 'text/plain' # the lack of a Content-Length header makes the outputter # send a 'Transfer-Encoding: chunked' response request.push(tail_f_producer(request, logfile, 1024)) request.done()
def execute_multi(self, calls, timeout): multi = pycurl.CurlMulti() for request in calls: multi.add_handle(request._curl) while True: while True: ret, num = multi.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break if num == 0: break if multi.select(timeout) < 0: raise pycurl.error(pycurl.E_OPERATION_TIMEOUTED) failed_calls = [] for request in calls: multi.remove_handle(request._curl) request._response_content = request._process_http_request() if request.response_code() == 0: failed_calls.append(request) else: if request._response_content: request.process() error_string = request.error() if error_string: self._errors.append(error_string) multi.close() return failed_calls
def continue_request (self, data, request): logger = self.supervisord.options.logger try: params, method = self.loads(data) # no <methodName> in the request or name is an empty string if not method: logger.trace('XML-RPC request received with no method name') request.error(400) return # we allow xml-rpc clients that do not send empty <params> # when there are no parameters for the method call if params is None: params = () try: logger.trace('XML-RPC method called: %s()' % method) value = self.call(method, params) # application-specific: instead of we never want to # marshal None (even though we could by saying allow_none=True # in dumps within xmlrpc_marshall), this is meant as # a debugging fixture, see issue 223. assert value is not None, ( 'return value from method %r with params %r is None' % (method, params) ) logger.trace('XML-RPC method %s() returned successfully' % method) except RPCError as err: # turn RPCError reported by method into a Fault instance value = xmlrpc.client.Fault(err.code, err.text) logger.trace('XML-RPC method %s() returned fault: [%d] %s' % ( method, err.code, err.text)) if isinstance(value, types.FunctionType): # returning a function from an RPC method implies that # this needs to be a deferred response (it needs to block). pushproducer = request.channel.push_with_producer pushproducer(DeferredXMLRPCResponse(request, value)) else: # if we get anything but a function, it implies that this # response doesn't need to be deferred, we can service it # right away. body = xmlrpc_marshal(value) request['Content-Type'] = 'text/xml' request['Content-Length'] = len(body) request.push(body) request.done() except: io = StringIO() traceback.print_exc(file=io) val = io.getvalue() logger.critical(val) # internal error, report as HTTP server error request.error(500)