def continue_request(self, data, request):
logger = self.supervisord.options.logger
try:
params, method = self.loads(data)
# no <methodName> in the request or name is an empty string
if not method:
logger.trace('XML-RPC request received with no method name')
request.error(400)
return
# we allow xml-rpc clients that do not send empty <params>
# when there are no parameters for the method call
if params is None:
params = ()
try:
logger.trace('XML-RPC method called: %s()' % method)
value = self.call(method, params)
# application-specific: instead of we never want to
# marshal None (even though we could by saying allow_none=True
# in dumps within xmlrpc_marshall), this is meant as
# a debugging fixture, see issue 223.
assert value is not None, (
'return value from method %r with params %r is None' %
(method, params))
logger.trace('XML-RPC method %s() returned successfully' %
method)
except RPCError as err:
# turn RPCError reported by method into a Fault instance
value = xmlrpc.client.Fault(err.code, err.text)
logger.trace('XML-RPC method %s() returned fault: [%d] %s' %
(method, err.code, err.text))
if isinstance(value, types.FunctionType):
# returning a function from an RPC method implies that
# this needs to be a deferred response (it needs to block).
pushproducer = request.channel.push_with_producer
pushproducer(DeferredXMLRPCResponse(request, value))
else:
# if we get anything but a function, it implies that this
# response doesn't need to be deferred, we can service it
# right away.
body = xmlrpc_marshal(value)
request['Content-Type'] = 'text/xml'
request['Content-Length'] = len(body)
request.push(body)
request.done()
except:
io = StringIO()
traceback.print_exc(file=io)
val = io.getvalue()
logger.critical(val)
# internal error, report as HTTP server error
request.error(500)
def submit_external_job(request, application_path):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if application_path.startswith('abfs:/') and not application_path.startswith('abfs://'):
application_path = application_path.replace("abfs:/", "abfs://")
elif application_path.startswith('s3a:/') and not application_path.startswith('s3a://'):
application_path = application_path.replace('s3a:/', 's3a://')
else:
application_path = "/" + application_path
if application_path.startswith("abfs://"):
application_path = abfspath(application_path)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
if params_form.is_valid():
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
mapping['dryrun'] = request.POST.get('dryrun_checkbox') == 'on'
application_name = os.path.basename(application_path)
application_class = Bundle if application_name == 'bundle.xml' else Coordinator if application_name == 'coordinator.xml' else get_workflow()
mapping[application_class.get_application_path_key()] = os.path.dirname(application_path)
try:
submission = Submission(request.user, fs=request.fs, jt=request.jt, properties=mapping)
job_id = submission.run(application_path)
except RestException as ex:
detail = ex._headers.get('oozie-error-message', ex)
if 'Max retries exceeded with url' in str(detail):
detail = '%s: %s' % (_('The Oozie server is not running'), detail)
LOG.exception(smart_str(detail))
raise PopupException(_("Error submitting job %s") % (application_path,), detail=detail)
jsonify = request.POST.get('format') == 'json'
if jsonify:
return JsonResponse({'status': 0, 'job_id': job_id, 'type': 'external_workflow'}, safe=False)
else:
request.info(_('Oozie job submitted'))
view = 'list_oozie_bundle' if application_name == 'bundle.xml' else 'list_oozie_coordinator' if application_name == 'coordinator.xml' else 'list_oozie_workflow'
return redirect(reverse('oozie:%s' % view, kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % params_form.errors))
else:
parameters = Submission(request.user, fs=request.fs, jt=request.jt).get_external_parameters(application_path)
initial_params = ParameterForm.get_initial_params(parameters)
params_form = ParametersFormSet(initial=initial_params)
popup = render('editor/submit_job_popup.mako', request, {
'params_form': params_form,
'name': _('Job'),
'action': reverse('oozie:submit_external_job', kwargs={'application_path': application_path}),
'show_dryrun': os.path.basename(application_path) != 'bundle.xml',
'return_json': request.GET.get('format') == 'json'
}, force_template=True).content
return JsonResponse(popup, safe=False)
def rerun_oozie_coordinator(request, job_id, app_path=None):
oozie_coordinator = check_job_access_permission(request, job_id)
check_job_edition_permission(oozie_coordinator, request.user)
ParametersFormSet = formset_factory(ParameterForm, extra=0)
if app_path is None:
app_path = oozie_coordinator.coordJobPath
else:
app_path = urllib.parse.unquote(app_path)
return_json = request.GET.get('format') == 'json'
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
rerun_form = RerunCoordForm(request.POST, oozie_coordinator=oozie_coordinator)
if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
args = {}
args['deployment_dir'] = app_path
params = {
'type': 'action',
'scope': ','.join(oozie_coordinator.aggreate(rerun_form.cleaned_data['actions'])),
'refresh': rerun_form.cleaned_data['refresh'],
'nocleanup': rerun_form.cleaned_data['nocleanup'],
}
properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
_rerun_coordinator(request, job_id, args, params, properties)
if rerun_form.cleaned_data['return_json']:
return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
else:
request.info(_('Coordinator re-running.'))
return redirect(reverse('oozie:list_oozie_coordinator', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s') % smart_unicode(rerun_form.errors))
return list_oozie_coordinator(request, job_id)
else:
rerun_form = RerunCoordForm(oozie_coordinator=oozie_coordinator, return_json=return_json)
initial_params = ParameterForm.get_initial_params(oozie_coordinator.conf_dict)
params_form = ParametersFormSet(initial=initial_params)
return render('dashboard/rerun_coord_popup.mako', request, {
'rerun_form': rerun_form,
'params_form': params_form,
'action': reverse('oozie:rerun_oozie_coord', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
'return_json': return_json,
'is_mini': request.GET.get('is_mini', False),
}, force_template=True)
def rerun_oozie_bundle(request, job_id, app_path):
oozie_bundle = check_job_access_permission(request, job_id)
check_job_edition_permission(oozie_bundle, request.user)
ParametersFormSet = formset_factory(ParameterForm, extra=0)
app_path = urllib.parse.unquote(app_path)
if request.method == 'POST':
params_form = ParametersFormSet(request.POST)
rerun_form = RerunBundleForm(request.POST, oozie_bundle=oozie_bundle)
if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
args = {}
args['deployment_dir'] = app_path
params = {
'coord-scope': ','.join(rerun_form.cleaned_data['coordinators']),
'refresh': rerun_form.cleaned_data['refresh'],
'nocleanup': rerun_form.cleaned_data['nocleanup'],
}
if rerun_form.cleaned_data['start'] and rerun_form.cleaned_data['end']:
date = {
'date-scope':
'%(start)s::%(end)s' % {
'start': utc_datetime_format(rerun_form.cleaned_data['start']),
'end': utc_datetime_format(rerun_form.cleaned_data['end'])
}
}
params.update(date)
properties = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
_rerun_bundle(request, job_id, args, params, properties)
request.info(_('Bundle re-running.'))
return redirect(reverse('oozie:list_oozie_bundle', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s' % (rerun_form.errors,)))
return list_oozie_bundle(request, job_id)
else:
rerun_form = RerunBundleForm(oozie_bundle=oozie_bundle)
initial_params = ParameterForm.get_initial_params(oozie_bundle.conf_dict)
params_form = ParametersFormSet(initial=initial_params)
return render('dashboard/rerun_bundle_popup.mako', request, {
'rerun_form': rerun_form,
'params_form': params_form,
'action': reverse('oozie:rerun_oozie_bundle', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS)}),
}, force_template=True)
def rerun_oozie_job(request, job_id, app_path=None):
ParametersFormSet = formset_factory(ParameterForm, extra=0)
oozie_workflow = check_job_access_permission(request, job_id)
check_job_edition_permission(oozie_workflow, request.user)
if app_path is None:
app_path = oozie_workflow.appPath
else:
app_path = urllib.parse.unquote(app_path)
return_json = request.GET.get('format') == 'json'
if request.method == 'POST':
rerun_form = RerunForm(request.POST, oozie_workflow=oozie_workflow)
params_form = ParametersFormSet(request.POST)
if sum([rerun_form.is_valid(), params_form.is_valid()]) == 2:
args = {}
if request.POST.get('rerun_form_choice') == 'fail_nodes':
args['fail_nodes'] = 'true'
else:
args['skip_nodes'] = ','.join(rerun_form.cleaned_data['skip_nodes'])
args['deployment_dir'] = app_path
mapping = dict([(param['name'], param['value']) for param in params_form.cleaned_data])
_rerun_workflow(request, job_id, args, mapping)
if rerun_form.cleaned_data['return_json']:
return JsonResponse({'status': 0, 'job_id': job_id}, safe=False)
else:
request.info(_('Workflow re-running.'))
return redirect(reverse('oozie:list_oozie_workflow', kwargs={'job_id': job_id}))
else:
request.error(_('Invalid submission form: %s %s' % (rerun_form.errors, params_form.errors)))
else:
rerun_form = RerunForm(oozie_workflow=oozie_workflow, return_json=return_json)
initial_params = ParameterForm.get_initial_params(oozie_workflow.conf_dict)
params_form = ParametersFormSet(initial=initial_params)
return render('dashboard/rerun_workflow_popup.mako', request, {
'rerun_form': rerun_form,
'params_form': params_form,
'action': reverse('oozie:rerun_oozie_job', kwargs={'job_id': job_id, 'app_path': urllib.parse.quote(app_path.encode('utf-8'), safe=SAFE_CHARACTERS_URI_COMPONENTS) }),
'return_json': return_json,
'is_mini': request.GET.get('is_mini', False),
}, force_template=True)
def handle_request(self, request):
if request.command != 'GET':
request.error (400) # bad request
return
path, params, query, fragment = request.split_uri()
if '%' in path:
path = http_server.unquote(path)
# strip off all leading slashes
while path and path[0] == '/':
path = path[1:]
path, process_name_and_channel = path.split('/', 1)
try:
process_name, channel = process_name_and_channel.split('/', 1)
except ValueError:
# no channel specified, default channel to stdout
process_name = process_name_and_channel
channel = 'stdout'
from supervisor.options import split_namespec
group_name, process_name = split_namespec(process_name)
group = self.supervisord.process_groups.get(group_name)
if group is None:
request.error(404) # not found
return
process = group.processes.get(process_name)
if process is None:
request.error(404) # not found
return
logfile = getattr(process.config, '%s_logfile' % channel, None)
if logfile is None or not os.path.exists(logfile):
# XXX problematic: processes that don't start won't have a log
# file and we probably don't want to go into fatal state if we try
# to read the log of a process that did not start.
request.error(410) # gone
return
mtime = os.stat(logfile)[stat.ST_MTIME]
request['Last-Modified'] = http_date.build_http_date(mtime)
request['Content-Type'] = 'text/plain'
# the lack of a Content-Length header makes the outputter
# send a 'Transfer-Encoding: chunked' response
request.push(tail_f_producer(request, logfile, 1024))
request.done()
def handle_request(self, request):
if request.command != 'GET':
request.error(400) # bad request
return
path, params, query, fragment = request.split_uri()
if '%' in path:
path = http_server.unquote(path)
# strip off all leading slashes
while path and path[0] == '/':
path = path[1:]
path, process_name_and_channel = path.split('/', 1)
try:
process_name, channel = process_name_and_channel.split('/', 1)
except ValueError:
# no channel specified, default channel to stdout
process_name = process_name_and_channel
channel = 'stdout'
from supervisor.options import split_namespec
group_name, process_name = split_namespec(process_name)
group = self.supervisord.process_groups.get(group_name)
if group is None:
request.error(404) # not found
return
process = group.processes.get(process_name)
if process is None:
request.error(404) # not found
return
logfile = getattr(process.config, '%s_logfile' % channel, None)
if logfile is None or not os.path.exists(logfile):
# XXX problematic: processes that don't start won't have a log
# file and we probably don't want to go into fatal state if we try
# to read the log of a process that did not start.
request.error(410) # gone
return
mtime = os.stat(logfile)[stat.ST_MTIME]
request['Last-Modified'] = http_date.build_http_date(mtime)
request['Content-Type'] = 'text/plain'
# the lack of a Content-Length header makes the outputter
# send a 'Transfer-Encoding: chunked' response
request.push(tail_f_producer(request, logfile, 1024))
request.done()
def handle_request(self, request):
if request.command != 'GET':
request.error (400) # bad request
return
logfile = self.supervisord.options.logfile
if logfile is None or not os.path.exists(logfile):
request.error(410) # gone
return
mtime = os.stat(logfile)[stat.ST_MTIME]
request['Last-Modified'] = http_date.build_http_date(mtime)
request['Content-Type'] = 'text/plain'
# the lack of a Content-Length header makes the outputter
# send a 'Transfer-Encoding: chunked' response
request.push(tail_f_producer(request, logfile, 1024))
request.done()
def handle_request(self, request):
if request.command != 'GET':
request.error(400) # bad request
return
logfile = self.supervisord.options.logfile
if logfile is None or not os.path.exists(logfile):
request.error(410) # gone
return
mtime = os.stat(logfile)[stat.ST_MTIME]
request['Last-Modified'] = http_date.build_http_date(mtime)
request['Content-Type'] = 'text/plain'
# the lack of a Content-Length header makes the outputter
# send a 'Transfer-Encoding: chunked' response
request.push(tail_f_producer(request, logfile, 1024))
request.done()
def execute_multi(self, calls, timeout):
multi = pycurl.CurlMulti()
for request in calls:
multi.add_handle(request._curl)
while True:
while True:
ret, num = multi.perform()
if ret != pycurl.E_CALL_MULTI_PERFORM:
break
if num == 0:
break
if multi.select(timeout) < 0:
raise pycurl.error(pycurl.E_OPERATION_TIMEOUTED)
failed_calls = []
for request in calls:
multi.remove_handle(request._curl)
request._response_content = request._process_http_request()
if request.response_code() == 0:
failed_calls.append(request)
else:
if request._response_content:
request.process()
error_string = request.error()
if error_string:
self._errors.append(error_string)
multi.close()
return failed_calls
def continue_request (self, data, request):
logger = self.supervisord.options.logger
try:
params, method = self.loads(data)
# no <methodName> in the request or name is an empty string
if not method:
logger.trace('XML-RPC request received with no method name')
request.error(400)
return
# we allow xml-rpc clients that do not send empty <params>
# when there are no parameters for the method call
if params is None:
params = ()
try:
logger.trace('XML-RPC method called: %s()' % method)
value = self.call(method, params)
# application-specific: instead of we never want to
# marshal None (even though we could by saying allow_none=True
# in dumps within xmlrpc_marshall), this is meant as
# a debugging fixture, see issue 223.
assert value is not None, (
'return value from method %r with params %r is None' %
(method, params)
)
logger.trace('XML-RPC method %s() returned successfully' %
method)
except RPCError as err:
# turn RPCError reported by method into a Fault instance
value = xmlrpc.client.Fault(err.code, err.text)
logger.trace('XML-RPC method %s() returned fault: [%d] %s' % (
method,
err.code, err.text))
if isinstance(value, types.FunctionType):
# returning a function from an RPC method implies that
# this needs to be a deferred response (it needs to block).
pushproducer = request.channel.push_with_producer
pushproducer(DeferredXMLRPCResponse(request, value))
else:
# if we get anything but a function, it implies that this
# response doesn't need to be deferred, we can service it
# right away.
body = xmlrpc_marshal(value)
request['Content-Type'] = 'text/xml'
request['Content-Length'] = len(body)
request.push(body)
request.done()
except:
io = StringIO()
traceback.print_exc(file=io)
val = io.getvalue()
logger.critical(val)
# internal error, report as HTTP server error
request.error(500)