def collect_incoming_data(self, data):
if PY3:
self.buffer = as_string(self.buffer) + as_string(data)
else:
self.buffer = self.buffer + data
if self.part==self.body:
self.feed(self.buffer)
self.buffer = ''
def test_emit_unicode_witherror(self):
handler = self._makeOne()
called = []
def fake_syslog(msg):
if not called:
called.append(msg)
raise UnicodeError
handler._syslog = fake_syslog
record = self._makeLogRecord(as_string('fií'))
handler.emit(record)
self.assertEqual(called, [as_string('fi\xc3\xad')])
def test_emit_unicode_witherror(self):
from supervisor.loggers import LevelsByName
handler = self._makeOne()
called = []
def fake_syslog(pri, msg):
if not called:
called.append((pri, msg))
raise UnicodeError
handler._syslog = fake_syslog
record = self._makeLogRecord(as_string('fií'))
handler.emit(record)
self.assertEqual(
called, [(LevelsByName.TRAC, as_string('fi\xc3\xad'))],
)
def _assertInState(self, *states):
if self.state not in states:
current_state = getProcessStateDescription(self.state)
allowable_states = ' '.join(map(getProcessStateDescription, states))
processname = as_string(self.config.name)
raise AssertionError('Assertion failed for %s: %s not in %s' % (
processname, current_state, allowable_states))
def test_emit_unicode_noerror(self):
handler = self._makeOne(self.filename)
record = self._makeLogRecord(as_string(b"fi\xc3\xad"))
handler.emit(record)
handler.close()
with open(self.filename, "rb") as f:
self.assertEqual(f.read(), b"fi\xc3\xad")
def test_fail(self):
from supervisor.childutils import listener
from supervisor.dispatchers import PEventListenerDispatcher
begin = as_string(PEventListenerDispatcher.RESULT_TOKEN_START)
stdout = StringIO()
listener.fail(stdout)
self.assertEqual(stdout.getvalue(), begin + '4\nFAIL')
def handle_request(self, request):
# authorize a request before handling it...
scheme = get_header(AUTHORIZATION, request.header)
if scheme:
scheme = scheme.lower()
if scheme == "basic":
cookie = get_header(AUTHORIZATION, request.header, 2)
try:
decoded = as_string(decodestring(as_bytes(cookie)))
except:
print_function("malformed authorization info <%s>" % cookie)
request.error(400)
return
auth_info = decoded.split(":", 1)
if self.authorizer.authorize(auth_info):
self.pass_count.increment()
request.auth_info = auth_info
self.handler.handle_request(request)
else:
self.handle_unauthorized(request)
# elif scheme == 'digest':
# print 'digest: ',AUTHORIZATION.group(2)
else:
print("unknown/unsupported auth method: %s" % scheme)
self.handle_unauthorized(request)
else:
# list both? prefer one or the other?
# you could also use a 'nonce' here. [see below]
# auth = 'Basic realm="%s" Digest realm="%s"' % (self.realm, self.realm)
# nonce = self.make_nonce (request)
# auth = 'Digest realm="%s" nonce="%s"' % (self.realm, nonce)
# request['WWW-Authenticate'] = auth
# print 'sending header: %s' % request['WWW-Authenticate']
self.handle_unauthorized(request)
def payload(self):
groupname = ''
if self.process.group is not None:
groupname = self.process.group.config.name
try:
data = as_string(self.data)
except UnicodeDecodeError:
data = 'Undecodable: %r' % self.data
# On Python 2, stuff needs to be in Unicode before invoking the
# % operator, otherwise implicit encodings to ASCII can cause
# failures
fmt = as_string('processname:%s groupname:%s pid:%s channel:%s\n%s')
result = fmt % (as_string(self.process.config.name),
as_string(groupname), self.pid,
as_string(self.channel), data)
return result
def feed(self, url, data):
try:
data = as_string(data)
except UnicodeDecodeError:
data = 'Undecodable: %r' % data
sys.stdout.write(data)
sys.stdout.flush()
def send(self, data, stdout=sys.stdout):
resultlen = len(data)
result = '%s%s\n%s' % (as_string(PEventListenerDispatcher.RESULT_TOKEN_START),
str(resultlen),
data)
stdout.write(result)
stdout.flush()
def request(self, host, handler, request_body, verbose=0):
if not self.connection:
self.connection = self._get_connection()
self.headers = {
"User-Agent" : self.user_agent,
"Content-Type" : "text/xml",
"Accept": "text/xml"
}
# basic auth
if self.username is not None and self.password is not None:
unencoded = "%s:%s" % (self.username, self.password)
encoded = as_string(encodestring(as_bytes(unencoded)))
encoded = encoded.replace('\n', '')
encoded = encoded.replace('\012', '')
self.headers["Authorization"] = "Basic %s" % encoded
self.headers["Content-Length"] = str(len(request_body))
self.connection.request('POST', handler, request_body, self.headers)
r = self.connection.getresponse()
if r.status != 200:
self.connection.close()
self.connection = None
raise xmlrpclib.ProtocolError(host + handler,
r.status,
r.reason,
'' )
data = r.read()
p, u = self.getparser()
p.feed(data)
p.close()
return u.close()
def test_token(self):
from supervisor.childutils import listener
from supervisor.dispatchers import PEventListenerDispatcher
token = as_string(PEventListenerDispatcher.READY_FOR_EVENTS_TOKEN)
stdout = StringIO()
listener.ready(stdout)
self.assertEqual(stdout.getvalue(), token)
def test_emit_unicode_noerror(self):
from supervisor.loggers import LevelsByName
handler = self._makeOne()
inp = as_string('fií')
record = self._makeLogRecord(inp)
handler.emit(record)
syslog.syslog.assert_called_with(LevelsByName.TRAC, 'fi\xc3\xad')
def _dispatchEvent(self, event):
pool_serial = event.pool_serials[self.config.name]
for process in self.processes.values():
if process.state != ProcessStates.RUNNING:
continue
if process.listener_state == EventListenerStates.READY:
processname = as_string(process.config.name)
payload = event.payload()
try:
event_type = event.__class__
serial = event.serial
envelope = self._eventEnvelope(event_type, serial,
pool_serial, payload)
process.write(as_bytes(envelope))
except OSError as why:
if why.args[0] != errno.EPIPE:
raise
self.config.options.logger.debug(
'epipe occurred while sending event %s '
'to listener %s, listener state unchanged' % (
event.serial, processname))
continue
process.listener_state = EventListenerStates.BUSY
process.event = event
self.config.options.logger.debug(
'event %s sent to listener %s' % (
event.serial, processname))
return True
return False
def _acceptEvent(self, event, head=False):
# events are required to be instances
# this has a side effect to fail with an attribute error on 'old style'
# classes
processname = as_string(self.config.name)
if not hasattr(event, 'serial'):
event.serial = new_serial(GlobalSerial)
if not hasattr(event, 'pool_serials'):
event.pool_serials = {}
if self.config.name not in event.pool_serials:
event.pool_serials[self.config.name] = new_serial(self)
else:
self.config.options.logger.debug(
'rebuffering event %s for pool %s (buf size=%d, max=%d)' % (
(event.serial, processname, len(self.event_buffer),
self.config.buffer_size)))
if len(self.event_buffer) >= self.config.buffer_size:
if self.event_buffer:
# discard the oldest event
discarded_event = self.event_buffer.pop(0)
self.config.options.logger.error(
'pool %s event buffer overflowed, discarding event %s' % (
(processname, discarded_event.serial)))
if head:
self.event_buffer.insert(0, event)
else:
self.event_buffer.append(event)
def test_handle_more_follow_file_recreated(self):
request = DummyRequest('/logtail/foo', None, None, None)
f = tempfile.NamedTemporaryFile()
f.write(as_bytes('a' * 80))
f.flush()
producer = self._makeOne(request, f.name, 80)
result = producer.more()
self.assertEqual(result, as_string('a' * 80))
f.close()
f2 = open(f.name, 'wb')
try:
f2.write(as_bytes('b' * 80))
f2.close()
result = producer.more()
finally:
os.unlink(f2.name)
self.assertEqual(result, as_string('b' * 80))
def test_handle_request_does_not_authorize_bad_credentials(self):
request = DummyRequest('/logtail/process1', None, None, None)
encoded = base64.b64encode(as_bytes("wrong:wrong"))
request.header = ["Authorization: Basic %s" % as_string(encoded)]
handler = DummyHandler()
auth_handler = self._makeOne({'user':'******'}, handler)
auth_handler.handle_request(request)
self.assertFalse(handler.handled_request)
def stop_report(self):
""" Log a 'waiting for x to stop' message with throttling. """
if self.state == ProcessStates.STOPPING:
now = time.time()
if now > (self.laststopreport + 2): # every 2 seconds
self.config.options.logger.info(
'waiting for %s to stop' % as_string(self.config.name))
self.laststopreport = now
def __repr__(self):
# repr can't return anything other than a native string,
# but the name might be unicode - a problem on Python 2.
name = self.config.name
if PY2:
name = as_string(name).encode('unicode-escape')
return '<%s instance at %s named %s>' % (self.__class__, id(self),
name)
def test_handle_more_follow_file_gone(self):
request = DummyRequest('/logtail/foo', None, None, None)
filename = tempfile.mktemp()
with open(filename, 'wb') as f:
f.write(as_bytes('a' * 80))
try:
producer = self._makeOne(request, f.name, 80)
finally:
os.unlink(f.name)
result = producer.more()
self.assertEqual(result, as_string('a' * 80))
with open(filename, 'wb') as f:
f.write(as_bytes('b' * 80))
try:
result = producer.more() # should open in new file
self.assertEqual(result, as_string('b' * 80))
finally:
os.unlink(f.name)
def test_handle_request_authorizes_good_password_with_colon(self):
request = DummyRequest('/logtail/process1', None, None, None)
# password contains colon
encoded = base64.b64encode(as_bytes("user:pass:word"))
request.header = ["Authorization: Basic %s" % as_string(encoded)]
handler = DummyHandler()
auth_handler = self._makeOne({'user':'******'}, handler)
auth_handler.handle_request(request)
self.assertTrue(handler.handled_request)
def test_send(self):
from supervisor.childutils import listener
from supervisor.dispatchers import PEventListenerDispatcher
begin = as_string(PEventListenerDispatcher.RESULT_TOKEN_START)
stdout = StringIO()
msg = 'the body data ya fool\n'
listener.send(msg, stdout)
expected = '%s%s\n%s' % (begin, len(msg), msg)
self.assertEqual(stdout.getvalue(), expected)
def __repr__(self):
# repr can't return anything other than a native string,
# but the name might be unicode - a problem on Python 2.
name = self.config.name
if PY2:
name = as_string(name).encode('unicode-escape')
return '<Subprocess at %s with name %s in state %s>' % (
id(self),
name,
getProcessStateDescription(self.get_state()))
def _spawn_as_parent(self, pid):
# Parent
self.pid = pid
options = self.config.options
options.close_child_pipes(self.pipes)
options.logger.info('spawned: \'%s\' with pid %s' % (as_string(self.config.name), pid))
self.spawnerr = None
self.delay = time.time() + self.config.startsecs
options.pidhistory[pid] = self
return pid
def test_handle_more(self):
request = DummyRequest('/logtail/foo', None, None, None)
from supervisor import http
f = tempfile.NamedTemporaryFile()
f.write(as_bytes('a' * 80))
f.flush()
producer = self._makeOne(request, f.name, 80)
result = producer.more()
self.assertEqual(result, as_string('a' * 80))
f.write(as_bytes('w' * 100))
f.flush()
result = producer.more()
self.assertEqual(result, as_string('w' * 100))
result = producer.more()
self.assertEqual(result, http.NOT_DONE_YET)
f.truncate(0)
f.flush()
result = producer.more()
self.assertEqual(result, '==> File truncated <==\n')
def payload(self):
groupname = ''
if self.process.group is not None:
groupname = self.process.group.config.name
try:
data = as_string(self.data)
except UnicodeDecodeError:
data = 'Undecodable: %r' % self.data
return 'processname:%s groupname:%s pid:%s\n%s' % (
self.process.config.name,
groupname,
self.pid,
data)
def __call__(self):
body = self.render()
if body is NOT_DONE_YET:
return NOT_DONE_YET
response = self.context.response
headers = response["headers"]
headers["Content-Type"] = self.content_type
headers["Pragma"] = "no-cache"
headers["Cache-Control"] = "no-cache"
headers["Expires"] = http_date.build_http_date(0)
response["body"] = as_string(body)
return response
def __call__(self):
body = self.render()
if body is NOT_DONE_YET:
return NOT_DONE_YET
response = self.context.response
headers = response['headers']
headers['Content-Type'] = self.content_type
headers['Pragma'] = 'no-cache'
headers['Cache-Control'] = 'no-cache'
headers['Expires'] = http_date.build_http_date(0)
response['body'] = as_string(body)
return response
def _readProcessLog(self, name, offset, length, channel):
group, process = self._getGroupAndProcess(name)
logfile = getattr(process.config, '%s_logfile' % channel)
if logfile is None or not os.path.exists(logfile):
raise RPCError(Faults.NO_FILE, logfile)
try:
return as_string(readFile(logfile, int(offset), int(length)))
except ValueError as inst:
why = inst.args[0]
raise RPCError(getattr(Faults, why))
def handle_connect(self):
self.connected = 1
method = "GET"
version = "HTTP/1.1"
self.push("%s %s %s" % (method, self.path, version))
self.push(CRLF)
self.header("Host", self.host)
self.header('Accept-Encoding', 'chunked')
self.header('Accept', '*/*')
self.header('User-agent', self.user_agent)
if self.password:
auth = '%s:%s' % (self.username, self.password)
auth = as_string(encodestring(as_bytes(auth))).strip()
self.header('Authorization', 'Basic %s' % auth)
self.push(CRLF)
self.push(CRLF)
def found_terminator(self):
""" We only override this to use 'deferring_http_request' class
instead of the normal http_request class; it sucks to need to override
this """
if self.current_request:
self.current_request.found_terminator()
else:
# we convert the header to text to facilitate processing.
# some of the underlying APIs (such as splitquery)
# expect text rather than bytes.
header = as_string(self.in_buffer)
self.in_buffer = b''
lines = header.split('\r\n')
# --------------------------------------------------
# crack the request header
# --------------------------------------------------
while lines and not lines[0]:
# as per the suggestion of http-1.1 section 4.1, (and
# Eric Parker <*****@*****.**>), ignore a leading
# blank lines (buggy browsers tack it onto the end of
# POST requests)
lines = lines[1:]
if not lines:
self.close_when_done()
return
request = lines[0]
command, uri, version = http_server.crack_request(request)
header = http_server.join_headers(lines[1:])
# unquote path if necessary (thanks to Skip Montanaro for pointing
# out that we must unquote in piecemeal fashion).
rpath, rquery = http_server.splitquery(uri)
if '%' in rpath:
if rquery:
uri = http_server.unquote(rpath) + '?' + rquery
else:
uri = http_server.unquote(rpath)
r = deferring_http_request(self, request, command, uri, version,
header)
self.request_counter.increment()
self.server.total_requests.increment()
if command is None:
self.log_info('Bad HTTP request: %s' % repr(request), 'error')
r.error(400)
return
# --------------------------------------------------
# handler selection and dispatch
# --------------------------------------------------
for h in self.server.handlers:
if h.match(r):
try:
self.current_request = r
# This isn't used anywhere.
# r.handler = h # CYCLE
h.handle_request(r)
except:
self.server.exceptions.increment()
(file, fun, line), t, v, tbinfo = \
asyncore.compact_traceback()
self.server.log_info(
'Server Error: %s, %s: file: %s line: %s' %
(t, v, file, line), 'error')
try:
r.error(500)
except:
pass
return
# no handlers, so complain
r.error(404)
def main(out=sys.stdout):
config = pkg_resources.resource_string(__name__, 'skel/sample.conf')
out.write(as_string(config))
def handle_listener_state_change(self):
data = self.state_buffer
if not data:
return
process = self.process
procname = process.config.name
state = process.listener_state
if state == EventListenerStates.UNKNOWN:
# this is a fatal state
self.state_buffer = b''
return
if state == EventListenerStates.ACKNOWLEDGED:
if len(data) < self.READY_FOR_EVENTS_LEN:
# not enough info to make a decision
return
elif data.startswith(self.READY_FOR_EVENTS_TOKEN):
self._change_listener_state(EventListenerStates.READY)
tokenlen = self.READY_FOR_EVENTS_LEN
self.state_buffer = self.state_buffer[tokenlen:]
process.event = None
else:
self._change_listener_state(EventListenerStates.UNKNOWN)
self.state_buffer = b''
process.event = None
if self.state_buffer:
# keep going til its too short
self.handle_listener_state_change()
else:
return
elif state == EventListenerStates.READY:
# the process sent some spurious data, be strict about it
self._change_listener_state(EventListenerStates.UNKNOWN)
self.state_buffer = b''
process.event = None
return
elif state == EventListenerStates.BUSY:
if self.resultlen is None:
# we haven't begun gathering result data yet
pos = data.find(b'\n')
if pos == -1:
# we can't make a determination yet, we dont have a full
# results line
return
result_line = self.state_buffer[:pos]
self.state_buffer = self.state_buffer[pos + 1:] # rid LF
resultlen = result_line[self.RESULT_TOKEN_START_LEN:]
try:
self.resultlen = int(resultlen)
except ValueError:
try:
result_line = as_string(result_line)
except UnicodeDecodeError:
result_line = 'Undecodable: %r' % result_line
process.config.options.logger.warn(
'%s: bad result line: \'%s\'' %
(procname, result_line))
self._change_listener_state(EventListenerStates.UNKNOWN)
self.state_buffer = b''
notify(EventRejectedEvent(process, process.event))
process.event = None
return
else:
needed = self.resultlen - len(self.result)
if needed:
self.result += self.state_buffer[:needed]
self.state_buffer = self.state_buffer[needed:]
needed = self.resultlen - len(self.result)
if not needed:
self.handle_result(self.result)
self.process.event = None
self.result = b''
self.resultlen = None
if self.state_buffer:
# keep going til its too short
self.handle_listener_state_change()
def recv(self, buffersize):
return as_string(os.read(self.fd, buffersize))
def spawn(self):
"""Start the subprocess. It must not be running already.
Return the process id. If the fork() call fails, return None.
"""
options = self.config.options
processname = as_string(self.config.name)
if self.pid:
msg = 'process \'%s\' already running' % processname
options.logger.warn(msg)
return
self.killing = False
self.spawnerr = None
self.exitstatus = None
self.system_stop = False
self.administrative_stop = False
self.laststart = time.time()
self._assertInState(ProcessStates.EXITED, ProcessStates.FATAL,
ProcessStates.BACKOFF, ProcessStates.STOPPED)
self.change_state(ProcessStates.STARTING)
try:
filename, argv = self.get_execv_args()
except ProcessException as what:
self.record_spawnerr(what.args[0])
self._assertInState(ProcessStates.STARTING)
self.change_state(ProcessStates.BACKOFF)
return
try:
self.dispatchers, self.pipes = self.config.make_dispatchers(self)
except (OSError, IOError) as why:
code = why.args[0]
if code == errno.EMFILE:
# too many file descriptors open
msg = 'too many open files to spawn \'%s\'' % processname
else:
msg = 'unknown error making dispatchers for \'%s\': %s' % (
processname, errno.errorcode.get(code, code))
self.record_spawnerr(msg)
self._assertInState(ProcessStates.STARTING)
self.change_state(ProcessStates.BACKOFF)
return
try:
pid = options.fork()
except OSError as why:
code = why.args[0]
if code == errno.EAGAIN:
# process table full
msg = ('Too many processes in process table to spawn \'%s\'' %
processname)
else:
msg = 'unknown error during fork for \'%s\': %s' % (
processname, errno.errorcode.get(code, code))
self.record_spawnerr(msg)
self._assertInState(ProcessStates.STARTING)
self.change_state(ProcessStates.BACKOFF)
options.close_parent_pipes(self.pipes)
options.close_child_pipes(self.pipes)
return
if pid != 0:
return self._spawn_as_parent(pid)
else:
return self._spawn_as_child(filename, argv)
def kill(self, sig):
"""Send a signal to the subprocess with the intention to kill
it (to make it exit). This may or may not actually kill it.
Return None if the signal was sent, or an error message string
if an error occurred or if the subprocess is not running.
"""
now = time.time()
options = self.config.options
processname = as_string(self.config.name)
# If the process is in BACKOFF and we want to stop or kill it, then
# BACKOFF -> STOPPED. This is needed because if startretries is a
# large number and the process isn't starting successfully, the stop
# request would be blocked for a long time waiting for the retries.
if self.state == ProcessStates.BACKOFF:
msg = ("Attempted to kill %s, which is in BACKOFF state." %
processname)
options.logger.debug(msg)
self.change_state(ProcessStates.STOPPED)
return None
if not self.pid:
msg = ("attempted to kill %s with sig %s but it wasn't running" %
(processname, signame(sig)))
options.logger.debug(msg)
return msg
# If we're in the stopping state, then we've already sent the stop
# signal and this is the kill signal
if self.state == ProcessStates.STOPPING:
killasgroup = self.config.killasgroup
else:
killasgroup = self.config.stopasgroup
as_group = ""
if killasgroup:
as_group = "process group "
options.logger.debug('killing %s (pid %s) %swith signal %s' %
(processname, self.pid, as_group, signame(sig)))
# RUNNING/STARTING/STOPPING -> STOPPING
self.killing = True
self.delay = now + self.config.stopwaitsecs
# we will already be in the STOPPING state if we're doing a
# SIGKILL as a result of overrunning stopwaitsecs
self._assertInState(ProcessStates.RUNNING, ProcessStates.STARTING,
ProcessStates.STOPPING)
self.change_state(ProcessStates.STOPPING)
pid = self.pid
if killasgroup:
# send to the whole process group instead
pid = -self.pid
try:
try:
options.kill(pid, sig)
except OSError as exc:
if exc.errno == errno.ESRCH:
msg = (
"unable to signal %s (pid %s), it probably just exited "
"on its own: %s" % (processname, self.pid, str(exc)))
options.logger.debug(msg)
# we could change the state here but we intentionally do
# not. we will do it during normal SIGCHLD processing.
return None
raise
except:
tb = traceback.format_exc()
msg = 'unknown problem killing %s (%s):%s' % (processname,
self.pid, tb)
options.logger.critical(msg)
self.change_state(ProcessStates.UNKNOWN)
self.killing = False
self.delay = 0
return msg
return None
def test_emit_unicode_noerror(self):
handler = self._makeOne()
inp = as_string('fií')
record = self._makeLogRecord(inp)
handler.emit(record)
syslog.syslog.assert_called_with('fi\xc3\xad')
def transition(self):
now = time.time()
state = self.state
self._check_and_adjust_for_system_clock_rollback(now)
logger = self.config.options.logger
if self.config.options.mood > SupervisorStates.RESTARTING:
# dont start any processes if supervisor is shutting down
if state == ProcessStates.EXITED:
if self.config.autorestart:
if self.config.autorestart is RestartUnconditionally:
# EXITED -> STARTING
self.spawn()
else: # autorestart is RestartWhenExitUnexpected
if self.exitstatus not in self.config.exitcodes:
# EXITED -> STARTING
self.spawn()
elif state == ProcessStates.STOPPED and not self.laststart:
if self.config.autostart:
# STOPPED -> STARTING
self.spawn()
elif state == ProcessStates.BACKOFF:
if self.backoff <= self.config.startretries:
if now > self.delay:
# BACKOFF -> STARTING
self.spawn()
processname = as_string(self.config.name)
if state == ProcessStates.STARTING:
if now - self.laststart > self.config.startsecs:
# STARTING -> RUNNING if the proc has started
# successfully and it has stayed up for at least
# proc.config.startsecs,
self.delay = 0
self.backoff = 0
self._assertInState(ProcessStates.STARTING)
self.change_state(ProcessStates.RUNNING)
msg = ('entered RUNNING state, process has stayed up for '
'> than %s seconds (startsecs)' % self.config.startsecs)
logger.info('success: %s %s' % (processname, msg))
if state == ProcessStates.BACKOFF:
if self.backoff > self.config.startretries:
# BACKOFF -> FATAL if the proc has exceeded its number
# of retries
self.give_up()
msg = ('entered FATAL state, too many start retries too '
'quickly')
logger.info('gave up: %s %s' % (processname, msg))
elif state == ProcessStates.STOPPING:
time_left = self.delay - now
if time_left <= 0:
# kill processes which are taking too long to stop with a final
# sigkill. if this doesn't kill it, the process will be stuck
# in the STOPPING state forever.
self.config.options.logger.warn(
'killing \'%s\' (%s) with SIGKILL' %
(processname, self.pid))
self.kill(signal.SIGKILL)
for child in element._children:
for el2 in melditerator(child, meldid):
nodeid = el2.attrib.get(_MELD_ID)
if nodeid is not None:
if meldid is None or nodeid == meldid:
yield el2
#-----------------------------------------------------------------------------
# Begin fork from Python 2.6.8 stdlib:
# - xml.elementtree.ElementTree._raise_serialization_error
# - xml.elementtree.ElementTree._encode_entity
# - xml.elementtree.ElementTree._namespace_map
# - xml.elementtree.ElementTree.fixtag
#-----------------------------------------------------------------------------
_NON_ASCII_MIN = as_string('\xc2\x80', 'utf-8') # u'\u0080'
_NON_ASCII_MAX = as_string('\xef\xbf\xbf', 'utf-8') # u'\uffff'
_escape_map = {
"&": "&",
"<": "<",
">": ">",
'"': """,
}
_namespace_map = {
# "well-known" namespace prefixes
"http://www.w3.org/XML/1998/namespace": "xml",
"http://www.w3.org/1999/xhtml": "html",
"http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf",
"http://schemas.xmlsoap.org/wsdl/": "wsdl",
def handle_data(self, data):
if isinstance(data, bytes):
data = as_string(data, self.encoding)
self.builder.data(data)
class supervisor_xmlrpc_handler(xmlrpc_handler):
path = '/RPC2'
IDENT = 'Supervisor XML-RPC Handler'
unmarshallers = {
"int": lambda x: int(x.text),
"i4": lambda x: int(x.text),
"boolean": lambda x: x.text == "1",
"string": lambda x: x.text or "",
"double": lambda x: float(x.text),
"dateTime.iso8601": lambda x: make_datetime(x.text),
"array": lambda x: x[0].text,
"data": lambda x: [v.text for v in x],
"struct": lambda x: dict([(k.text or "", v.text) for k, v in x]),
"base64": lambda x: as_string(decodestring(as_bytes(x.text or ""))),
"param": lambda x: x[0].text,
}
def __init__(self, supervisord, subinterfaces):
self.rpcinterface = RootRPCInterface(subinterfaces)
self.supervisord = supervisord
def loads(self, data):
params = method = None
for action, elem in iterparse(StringIO(data)):
unmarshall = self.unmarshallers.get(elem.tag)
if unmarshall:
data = unmarshall(elem)
elem.clear()
elem.text = data
elif elem.tag == "value":
try:
data = elem[0].text
except IndexError:
data = elem.text or ""
elem.clear()
elem.text = data
elif elem.tag == "methodName":
method = elem.text
elif elem.tag == "params":
params = tuple([v.text for v in elem])
return params, method
def match(self, request):
return request.uri.startswith(self.path)
def continue_request(self, data, request):
logger = self.supervisord.options.logger
try:
try:
params, method = self.loads(data)
except:
logger.error(
'XML-RPC request data %r is invalid: unmarshallable' %
(data, ))
request.error(400)
return
# no <methodName> in the request or name is an empty string
if not method:
logger.error(
'XML-RPC request data %r is invalid: no method name' %
(data, ))
request.error(400)
return
# we allow xml-rpc clients that do not send empty <params>
# when there are no parameters for the method call
if params is None:
params = ()
try:
logger.trace('XML-RPC method called: %s()' % method)
value = self.call(method, params)
logger.trace('XML-RPC method %s() returned successfully' %
method)
except RPCError as err:
# turn RPCError reported by method into a Fault instance
value = xmlrpclib.Fault(err.code, err.text)
logger.trace('XML-RPC method %s() returned fault: [%d] %s' %
(method, err.code, err.text))
if isinstance(value, types.FunctionType):
# returning a function from an RPC method implies that
# this needs to be a deferred response (it needs to block).
pushproducer = request.channel.push_with_producer
pushproducer(DeferredXMLRPCResponse(request, value))
else:
# if we get anything but a function, it implies that this
# response doesn't need to be deferred, we can service it
# right away.
body = xmlrpc_marshal(value)
request['Content-Type'] = 'text/xml'
request['Content-Length'] = len(body)
request.push(body)
request.done()
except:
tb = traceback.format_exc()
logger.critical(
"Handling XML-RPC request with data %r raised an unexpected "
"exception: %s" % (data, tb))
# internal error, report as HTTP server error
request.error(500)
def call(self, method, params):
return traverse(self.rpcinterface, method, params)
def finish(self, pid, sts):
""" The process was reaped and we need to report and manage its state
"""
self.drain()
es, msg = decode_wait_status(sts)
now = time.time()
self._check_and_adjust_for_system_clock_rollback(now)
self.laststop = now
processname = as_string(self.config.name)
if now > self.laststart:
too_quickly = now - self.laststart < self.config.startsecs
else:
too_quickly = False
self.config.options.logger.warn(
"process \'%s\' (%s) laststart time is in the future, don't "
"know how long process was running so assuming it did "
"not exit too quickly" % (processname, self.pid))
exit_expected = es in self.config.exitcodes
if self.killing:
# likely the result of a stop request
# implies STOPPING -> STOPPED
self.killing = False
self.delay = 0
self.exitstatus = es
msg = "stopped: %s (%s)" % (processname, msg)
self._assertInState(ProcessStates.STOPPING)
self.change_state(ProcessStates.STOPPED)
elif too_quickly:
# the program did not stay up long enough to make it to RUNNING
# implies STARTING -> BACKOFF
self.exitstatus = None
self.spawnerr = 'Exited too quickly (process log may have details)'
msg = "exited: %s (%s)" % (processname, msg + "; not expected")
self._assertInState(ProcessStates.STARTING)
self.change_state(ProcessStates.BACKOFF)
else:
# this finish was not the result of a stop request, the
# program was in the RUNNING state but exited
# implies RUNNING -> EXITED normally but see next comment
self.delay = 0
self.backoff = 0
self.exitstatus = es
# if the process was STARTING but a system time change causes
# self.laststart to be in the future, the normal STARTING->RUNNING
# transition can be subverted so we perform the transition here.
if self.state == ProcessStates.STARTING:
self.change_state(ProcessStates.RUNNING)
self._assertInState(ProcessStates.RUNNING)
if exit_expected:
# expected exit code
msg = "exited: %s (%s)" % (processname, msg + "; expected")
self.change_state(ProcessStates.EXITED, expected=True)
else:
# unexpected exit code
self.spawnerr = 'Bad exit code %s' % es
msg = "exited: %s (%s)" % (processname, msg + "; not expected")
self.change_state(ProcessStates.EXITED, expected=False)
self.config.options.logger.info(msg)
self.pid = 0
self.config.options.close_parent_pipes(self.pipes)
self.pipes = {}
self.dispatchers = {}
# if we died before we processed the current event (only happens
# if we're an event listener), notify the event system that this
# event was rejected so it can be processed again.
if self.event is not None:
# Note: this should only be true if we were in the BUSY
# state when finish() was called.
events.notify(events.EventRejectedEvent(self, self.event))
self.event = None
import datetime, time
def make_datetime(text):
return datetime.datetime(*time.strptime(text, "%Y%m%dT%H:%M:%S")[:6])
unmarshallers = {
"int": lambda x: int(x.text),
"i4": lambda x: int(x.text),
"boolean": lambda x: x.text == "1",
"string": lambda x: x.text or "",
"double": lambda x: float(x.text),
"dateTime.iso8601": lambda x: make_datetime(x.text),
"array": lambda x: x[0].text,
"data": lambda x: [v.text for v in x],
"struct": lambda x: dict([(k.text or "", v.text) for k, v in x]),
"base64": lambda x: as_string(decodestring(as_bytes(x.text or ""))),
"value": lambda x: x[0].text,
"param": lambda x: x[0].text,
}
def loads(data):
params = method = None
for action, elem in iterparse(StringIO(data)):
unmarshall = unmarshallers.get(elem.tag)
if unmarshall:
data = unmarshall(elem)
elem.clear()
elem.text = data
elif elem.tag == "methodName":
method = elem.text
elif elem.tag == "params":
def ready(self, stdout=sys.stdout):
stdout.write(as_string(
PEventListenerDispatcher.READY_FOR_EVENTS_TOKEN))
stdout.flush()
def render(self):
form = self.context.form
response = self.context.response
processname = form.get('processname')
action = form.get('action')
message = form.get('message')
if action:
if not self.callback:
self.callback = self.make_callback(processname, action)
return NOT_DONE_YET
else:
message = self.callback()
if message is NOT_DONE_YET:
return NOT_DONE_YET
if message is not None:
server_url = form['SERVER_URL']
location = server_url + '?message=%s' % urllib.quote(message)
response['headers']['Location'] = location
supervisord = self.context.supervisord
rpcinterface = RootRPCInterface(
[('supervisor', SupervisorNamespaceRPCInterface(supervisord))]
)
processnames = []
for group in supervisord.process_groups.values():
for gprocname in group.processes.keys():
processnames.append((group.config.name, gprocname))
processnames.sort()
data = []
for groupname, processname in processnames:
actions = self.actions_for_process(
supervisord.process_groups[groupname].processes[processname])
sent_name = make_namespec(groupname, processname)
info = rpcinterface.supervisor.getProcessInfo(sent_name)
data.append({
'status': info['statename'],
'name': processname,
'group': groupname,
'actions': actions,
'state': info['state'],
'description': info['description'],
})
root = self.clone()
if message is not None:
statusarea = root.findmeld('statusmessage')
statusarea.attrib['class'] = 'status_msg'
statusarea.content(message)
if data:
iterator = root.findmeld('tr').repeat(data)
shaded_tr = False
for tr_element, item in iterator:
status_text = tr_element.findmeld('status_text')
status_text.content(item['status'].lower())
status_text.attrib['class'] = self.css_class_for_state(
item['state'])
info_text = tr_element.findmeld('info_text')
info_text.content(item['description'])
anchor = tr_element.findmeld('name_anchor')
processname = make_namespec(item['group'], item['name'])
anchor.attributes(href='tail.html?processname=%s' %
urllib.quote(processname))
anchor.content(processname)
actions = item['actions']
actionitem_td = tr_element.findmeld('actionitem_td')
for li_element, actionitem in actionitem_td.repeat(actions):
anchor = li_element.findmeld('actionitem_anchor')
if actionitem is None:
anchor.attrib['class'] = 'hidden'
else:
anchor.attributes(href=actionitem['href'],
name=actionitem['name'])
anchor.content(actionitem['name'])
if actionitem['target']:
anchor.attributes(target=actionitem['target'])
if shaded_tr:
tr_element.attrib['class'] = 'shade'
shaded_tr = not shaded_tr
else:
table = root.findmeld('statustable')
table.replace('No programs to manage')
root.findmeld('supervisor_version').content(VERSION)
copyright_year = str(datetime.date.today().year)
root.findmeld('copyright_date').content(copyright_year)
return as_string(root.write_xhtmlstring())
class supervisor_xmlrpc_handler(xmlrpc_handler):
path = '/RPC2'
IDENT = 'Supervisor XML-RPC Handler'
unmarshallers = {
"int": lambda x: int(x.text),
"i4": lambda x: int(x.text),
"boolean": lambda x: x.text == "1",
"string": lambda x: x.text or "",
"double": lambda x: float(x.text),
"dateTime.iso8601": lambda x: make_datetime(x.text),
"array": lambda x: x[0].text,
"data": lambda x: [v.text for v in x],
"struct": lambda x: dict([(k.text or "", v.text) for k, v in x]),
"base64": lambda x: as_string(decodestring(as_bytes(x.text or ""))),
"param": lambda x: x[0].text,
}
def __init__(self, supervisord, subinterfaces):
# subinterfaces [('supervisor', < supervisor.rpcinterface.SupervisorNamespaceRPCInterface instance at 0x7f350f874b48 >),
# # ('system', < supervisor.xmlrpc.SystemNamespaceRPCInterface instance at 0x7f350f874b90 >)]
self.rpcinterface = RootRPCInterface(subinterfaces)
# self.rpcinterface.supervisor = SupervisorNamespaceRPCInterface instance
# self.rpcinterface.system = supervisor.xmlrpc.SystemNamespaceRPCInterface instance
self.supervisord = supervisord
def loads(self, data):
params = method = None
# python中用ElementTree.iterparse()读取xml文件中的多层节点
for action, elem in iterparse(StringIO(data)):
unmarshall = self.unmarshallers.get(elem.tag)
if unmarshall:
data = unmarshall(elem)
elem.clear()
elem.text = data
elif elem.tag == "value":
try:
data = elem[0].text
except IndexError:
data = elem.text or ""
elem.clear()
elem.text = data
elif elem.tag == "methodName":
method = elem.text
elif elem.tag == "params":
params = tuple([v.text for v in elem])
# print(params,method)
return params, method
def match(self, request):
return request.uri.startswith(self.path)
def continue_request(self, data, request):
# 正主
logger = self.supervisord.options.logger
# print(data)
try:
try:
# on 2.x, the Expat parser doesn't like Unicode which actually
# contains non-ASCII characters. It's a bit of a kludge to
# do it conditionally here, but it's down to how underlying
# libs behave
if PY2:
data = data.encode('ascii', 'xmlcharrefreplace')
params, method = self.loads(data)
# print(params,method)
except:
logger.error(
'XML-RPC request data %r is invalid: unmarshallable' %
(data, ))
request.error(400)
return
# no <methodName> in the request or name is an empty string
if not method:
logger.error(
'XML-RPC request data %r is invalid: no method name' %
(data, ))
request.error(400)
return
# we allow xml-rpc clients that do not send empty <params>
# when there are no parameters for the method call
if params is None:
params = ()
try:
logger.trace('XML-RPC method called: %s()' % method)
# call 是核心功能
# print(method, params)
value = self.call(method, params)
# print('value:',value)
logger.trace('XML-RPC method %s() returned successfully' %
method)
except RPCError as err:
# turn RPCError reported by method into a Fault instance
value = xmlrpclib.Fault(err.code, err.text)
logger.trace('XML-RPC method %s() returned fault: [%d] %s' %
(method, err.code, err.text))
if isinstance(value, types.FunctionType):
# returning a function from an RPC method implies that
# this needs to be a deferred response (it needs to block).
pushproducer = request.channel.push_with_producer
pushproducer(DeferredXMLRPCResponse(request, value))
else:
# if we get anything but a function, it implies that this
# response doesn't need to be deferred, we can service it
# right away.
body = as_bytes(xmlrpc_marshal(value))
request['Content-Type'] = 'text/xml'
request['Content-Length'] = len(body)
request.push(body)
request.done()
except:
tb = traceback.format_exc()
logger.critical(
"Handling XML-RPC request with data %r raised an unexpected "
"exception: %s" % (data, tb))
# internal error, report as HTTP server error
request.error(500)
def call(self, method, params):
return traverse(self.rpcinterface, method, params)
def write(self, msg):
if self.error:
error = self.error
self.error = None
raise error
self.written += as_string(msg)
def test_emit_unicode_noerror(self):
handler = self._makeOne(self.filename)
record = self._makeLogRecord(as_string(b'fi\xc3\xad'))
handler.emit(record)
with open(self.filename, 'rb') as f:
self.assertEqual(f.read(), b'fi\xc3\xad')
