def refill_buffer (self):
""" Implement deferreds """
while 1:
if len(self.producer_fifo):
p = self.producer_fifo.first()
# a 'None' in the producer fifo is a sentinel,
# telling us to close the channel.
if p is None:
if not self.ac_out_buffer:
self.producer_fifo.pop()
self.close()
return
elif isinstance(p, str):
self.producer_fifo.pop()
self.ac_out_buffer += p
return
data = p.more()
if data is NOT_DONE_YET:
self.delay = p.delay
return
elif data:
try:
self.ac_out_buffer = self.ac_out_buffer + data
except TypeError:
self.ac_out_buffer = as_bytes(self.ac_out_buffer) + as_bytes(data)
self.delay = False
return
else:
self.producer_fifo.pop()
else:
return
def __init__(self, config, state=None):
self.config = config
self.logsremoved = False
self.stop_called = False
self.stop_report_called = True
self.backoff_secs = None
self.spawned = False
if state is None:
from supervisor.process import ProcessStates
state = ProcessStates.RUNNING
self.state = state
self.error_at_clear = False
self.killed_with = None
self.drained = False
self.stdout_buffer = as_bytes("")
self.stderr_buffer = as_bytes("")
self.stdout_logged = as_bytes("")
self.stderr_logged = as_bytes("")
self.stdin_buffer = as_bytes("")
self.pipes = {}
self.rpipes = {}
self.dispatchers = {}
self.finished = None
self.logs_reopened = False
self.execv_arg_exception = None
self.input_fd_drained = None
self.output_fd_drained = None
self.transitioned = False
self.write_error = None
def test_handle_more_follow_file_recreated(self):
request = DummyRequest('/logtail/foo', None, None, None)
tmpfile = TempFileOpen()
try:
tmpfile.write(as_bytes('a' * 80))
tmpfile.flush()
producer = self._makeOne(request, tmpfile.name, 80)
result = producer.more()
self.assertEqual(result, as_bytes('a' * 80))
# in windows open files need to be closed
tmpfile.close()
tmpfile = open(tmpfile.name, 'wb')
try:
tmpfile.write(as_bytes('b' * 80))
tmpfile.flush()
result = producer.more()
finally:
tmpfile.close()
os.remove(tmpfile.name)
finally:
tmpfile.close()
self.assertEqual(result, as_bytes('b' * 80))
def more (self):
while len(self.buffer) < self.buffer_size:
data = self.producer.more()
if data is NOT_DONE_YET:
return NOT_DONE_YET
if data:
try:
self.buffer = self.buffer + data
except TypeError:
self.buffer = as_bytes(self.buffer) + as_bytes(data)
else:
break
r = self.buffer
self.buffer = ''
return r
def handle_request(self, request):
# authorize a request before handling it...
scheme = get_header(AUTHORIZATION, request.header)
if scheme:
scheme = scheme.lower()
if scheme == "basic":
cookie = get_header(AUTHORIZATION, request.header, 2)
try:
decoded = as_string(decodestring(as_bytes(cookie)))
except:
print_function("malformed authorization info <%s>" % cookie)
request.error(400)
return
auth_info = decoded.split(":", 1)
if self.authorizer.authorize(auth_info):
self.pass_count.increment()
request.auth_info = auth_info
self.handler.handle_request(request)
else:
self.handle_unauthorized(request)
# elif scheme == 'digest':
# print 'digest: ',AUTHORIZATION.group(2)
else:
print("unknown/unsupported auth method: %s" % scheme)
self.handle_unauthorized(request)
else:
# list both? prefer one or the other?
# you could also use a 'nonce' here. [see below]
# auth = 'Basic realm="%s" Digest realm="%s"' % (self.realm, self.realm)
# nonce = self.make_nonce (request)
# auth = 'Digest realm="%s" nonce="%s"' % (self.realm, nonce)
# request['WWW-Authenticate'] = auth
# print 'sending header: %s' % request['WWW-Authenticate']
self.handle_unauthorized(request)
def request(self, host, handler, request_body, verbose=0):
if not self.connection:
self.connection = self._get_connection()
self.headers = {
"User-Agent" : self.user_agent,
"Content-Type" : "text/xml",
"Accept": "text/xml"
}
# basic auth
if self.username is not None and self.password is not None:
unencoded = "%s:%s" % (self.username, self.password)
encoded = as_string(encodestring(as_bytes(unencoded)))
encoded = encoded.replace('\n', '')
encoded = encoded.replace('\012', '')
self.headers["Authorization"] = "Basic %s" % encoded
self.headers["Content-Length"] = str(len(request_body))
self.connection.request('POST', handler, request_body, self.headers)
r = self.connection.getresponse()
if r.status != 200:
self.connection.close()
self.connection = None
raise xmlrpclib.ProtocolError(host + handler,
r.status,
r.reason,
'' )
data = r.read()
p, u = self.getparser()
p.feed(data)
p.close()
return u.close()
def _dispatchEvent(self, event):
pool_serial = event.pool_serials[self.config.name]
for process in self.processes.values():
if process.state != ProcessStates.RUNNING:
continue
if process.listener_state == EventListenerStates.READY:
payload = str(event)
try:
event_type = event.__class__
serial = event.serial
envelope = self._eventEnvelope(event_type, serial,
pool_serial, payload)
process.write(as_bytes(envelope))
except OSError as why:
if why.args[0] != errno.EPIPE:
raise
continue
process.listener_state = EventListenerStates.BUSY
process.event = event
self.config.options.logger.debug(
'event %s sent to listener %s' % (
event.serial, process.config.name))
return True
return False
def push (self, thing):
# Sometimes, text gets pushed by XMLRPC logic for later
# processing.
if isinstance(thing, str):
thing = as_bytes(thing)
if isinstance(thing, bytes):
thing = producers.simple_producer(thing, buffer_size=len(thing))
self.outgoing.append(thing)
def test_handle_request_does_not_authorize_bad_credentials(self):
request = DummyRequest('/logtail/process1', None, None, None)
encoded = base64.b64encode(as_bytes("wrong:wrong"))
request.header = ["Authorization: Basic %s" % as_string(encoded)]
handler = DummyHandler()
auth_handler = self._makeOne({'user':'******'}, handler)
auth_handler.handle_request(request)
self.assertFalse(handler.handled_request)
def test_handle_more_follow_file_recreated(self):
request = DummyRequest('/logtail/foo', None, None, None)
f = tempfile.NamedTemporaryFile()
f.write(as_bytes('a' * 80))
f.flush()
producer = self._makeOne(request, f.name, 80)
result = producer.more()
self.assertEqual(result, as_string('a' * 80))
f.close()
f2 = open(f.name, 'wb')
try:
f2.write(as_bytes('b' * 80))
f2.close()
result = producer.more()
finally:
os.unlink(f2.name)
self.assertEqual(result, as_string('b' * 80))
def test_handle_request_authorizes_good_password_with_colon(self):
request = DummyRequest('/logtail/process1', None, None, None)
# password contains colon
encoded = base64.b64encode(as_bytes("user:pass:word"))
request.header = ["Authorization: Basic %s" % as_string(encoded)]
handler = DummyHandler()
auth_handler = self._makeOne({'user':'******'}, handler)
auth_handler.handle_request(request)
self.assertTrue(handler.handled_request)
def test_handle_more_fd_closed(self):
request = DummyRequest('/logtail/foo', None, None, None)
with tempfile.NamedTemporaryFile() as f:
f.write(as_bytes('a' * 80))
f.flush()
producer = self._makeOne(request, f.name, 80)
producer.file.close()
result = producer.more()
self.assertEqual(result, producer.more())
def pull_trigger(self, thunk=None):
# print 'PULL_TRIGGER: ', len(self.thunks)
if thunk:
self.lock.acquire()
try:
self.thunks.append(thunk)
finally:
self.lock.release()
os.write(self.trigger, as_bytes("x"))
def __call__(self, *args, **kwargs):
message = args[-1]
if sys.version_info < (3, 0) and isinstance(message, unicode):
# Python 2.x raises a UnicodeEncodeError when attempting to
# transmit unicode characters that don't encode in the
# default encoding.
args = list(args)
args[-1] = as_bytes(message)
super(MockSysLog, self).__call__(*args, **kwargs)
def test_handle_more_follow_file_gone(self):
request = DummyRequest('/logtail/foo', None, None, None)
filename = tempfile.mktemp()
with open(filename, 'wb') as f:
f.write(as_bytes('a' * 80))
try:
producer = self._makeOne(request, f.name, 80)
finally:
os.unlink(f.name)
result = producer.more()
self.assertEqual(result, as_string('a' * 80))
with open(filename, 'wb') as f:
f.write(as_bytes('b' * 80))
try:
result = producer.more() # should open in new file
self.assertEqual(result, as_string('b' * 80))
finally:
os.unlink(f.name)
def test_handle_more_follow_file_gone(self):
request = DummyRequest('/logtail/foo', None, None, None)
tmpfile = TempFileOpen()
tmpfile.write(as_bytes('a' * 80))
try:
producer = self._makeOne(request, tmpfile.name, 80)
producer.close()
finally:
tmpfile.close()
result = producer.more()
self.assertEqual(result, '')
with open(tmpfile.name, 'wb') as tmpfile:
tmpfile.write(as_bytes('b' * 80))
try:
result = producer.more() # should open in new file
self.assertEqual(result, as_bytes('b' * 80))
finally:
os.remove(tmpfile.name)
def test_handle_more(self):
request = DummyRequest('/logtail/foo', None, None, None)
from supervisor import http
f = tempfile.NamedTemporaryFile()
f.write(as_bytes('a' * 80))
f.flush()
producer = self._makeOne(request, f.name, 80)
result = producer.more()
self.assertEqual(result, as_string('a' * 80))
f.write(as_bytes('w' * 100))
f.flush()
result = producer.more()
self.assertEqual(result, as_string('w' * 100))
result = producer.more()
self.assertEqual(result, http.NOT_DONE_YET)
f.truncate(0)
f.flush()
result = producer.more()
self.assertEqual(result, '==> File truncated <==\n')
def handle_write_event(self):
if self.input_buffer:
try:
self.flush()
except OSError as why:
if why.args[0] == errno.EPIPE:
self.input_buffer = as_bytes('')
self.close()
else:
raise
def checkused(self, socketname):
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
s.connect(socketname)
s.send(as_bytes("GET / HTTP/1.0\r\n\r\n"))
s.recv(1)
s.close()
except socket.error:
return False
else:
return True
def authorize(self, auth_info):
username, password = auth_info
if username in self.dict:
stored_password = self.dict[username]
if stored_password.startswith('{SHA}'):
password_hash = sha1(as_bytes(password)).hexdigest()
return stored_password[5:] == password_hash
else:
return stored_password == password
else:
return False
def test_handle_more(self):
request = DummyRequest('/logtail/foo', None, None, None)
from supervisor import http
f = tempfile.NamedTemporaryFile()
f.write(as_bytes('a' * 80))
f.flush()
t = f.name
producer = self._makeOne(request, t, 80)
result = producer.more()
self.assertEqual(result, TAILF_LOG_WRAPPER.format(cgi.escape(as_bytes('a' * 80).decode('utf-8'))))
f.write(as_bytes('w' * 100))
f.flush()
result = producer.more()
self.assertEqual(result, TAILF_LOG_WRAPPER.format(cgi.escape(as_bytes('w' * 100).decode('utf-8'))))
result = producer.more()
self.assertEqual(result, http.NOT_DONE_YET)
f.truncate(0)
f.flush()
result = producer.more()
self.assertEqual(result, '==> File truncated <==\n')
def test_handle_more_fd_closed(self):
request = DummyRequest('/logtail/foo', None, None, None)
with TempFileOpen() as tmpfile:
tmpfile.write(as_bytes('a' * 80))
tmpfile.flush()
producer = self._makeOne(request, tmpfile.name, 80)
producer.close()
producer.more()
result = producer.more()
self.assertEqual(result, NOT_DONE_YET)
def handle_read(self):
try:
data = self.recv(self.ac_in_buffer_size)
except socket.error:
self.handle_error()
return
self.ac_in_buffer = as_bytes(self.ac_in_buffer) + as_bytes(data)
while self.ac_in_buffer:
lb = len(self.ac_in_buffer)
terminator = self.get_terminator()
if not terminator:
self.collect_incoming_data(self.ac_in_buffer)
self.ac_in_buffer = ''
elif isinstance(terminator, int) or isinstance(terminator, long):
n = terminator
if lb < n:
self.collect_incoming_data(self.ac_in_buffer)
self.ac_in_buffer = ''
self.terminator= lb
else:
self.collect_incoming_data(self.ac_in_buffer[:n])
self.ac_in_buffer = self.ac_in_buffer[n:]
self.terminator = 0
self.found_terminator()
else:
terminator_len = len(as_bytes(terminator))
index = self.ac_in_buffer.find(as_bytes(terminator))
if index !=1:
if index > 0:
self.collect_incoming_data(self.ac_in_buffer[:index])
self.ac_in_buffer = self.ac_in_buffer[index + terminator_len:]
self.found_terminator()
else:
self.collect_incoming_data(self.ac_in_buffer)
self.ac_in_buffer = ''
def more (self):
if self.producer:
result = self.producer.more()
if result is NOT_DONE_YET:
return NOT_DONE_YET
if not result:
self.producer = None
self.function (self.bytes)
else:
self.bytes += len(as_bytes(result))
return result
else:
return ''
def error (self, code):
self.reply_code = code
message = self.responses[code]
s = self.DEFAULT_ERROR_MESSAGE % {
'code': code,
'message': message,
}
s = as_bytes(s)
self['Content-Length'] = len(s)
self['Content-Type'] = 'text/html'
# make an error reply
self.push(s)
self.done()
def more (self):
if self.producer:
data = self.producer.more()
if data:
s = '%x' % len(data)
return as_bytes(s) + b'\r\n' + data + b'\r\n'
else:
self.producer = None
if self.footers:
return b'\r\n'.join([b'0'] + self.footers) + b'\r\n\r\n'
else:
return b'0\r\n\r\n'
else:
return b''
def test_handle_more(self):
request = DummyRequest('/logtail/foo', None, None, None)
from supervisor import http
with TempFileOpen() as tmpfile:
# test 1
tmpfile.write(as_bytes('a' * 80))
tmpfile.flush()
producer = self._makeOne(request, tmpfile.name, 80)
result = producer.more()
self.assertEqual(result, as_bytes('a' * 80))
# test 2
tmpfile.write(as_bytes('b' * 100))
tmpfile.flush()
result = producer.more()
self.assertEqual(result, as_bytes('b' * 100))
result = producer.more()
self.assertEqual(result, http.NOT_DONE_YET)
tmpfile.truncate(0)
tmpfile.flush()
result = producer.more()
self.assertEqual(result, '==> File truncated <==\n')
def more (self):
if self.producer:
data = self.producer.more()
if data is NOT_DONE_YET:
return NOT_DONE_YET
elif data:
return '%x\r\n%s\r\n' % (len(as_bytes(data)), data)
else:
self.producer = None
if self.footers:
return '\r\n'.join(['0'] + self.footers) + '\r\n\r\n'
else:
return '0\r\n\r\n'
else:
return ''
def handle_connect(self):
self.connected = 1
method = "GET"
version = "HTTP/1.1"
self.push("%s %s %s" % (method, self.path, version))
self.push(CRLF)
self.header("Host", self.host)
self.header('Accept-Encoding', 'chunked')
self.header('Accept', '*/*')
self.header('User-agent', self.user_agent)
if self.password:
auth = '%s:%s' % (self.username, self.password)
auth = as_string(encodestring(as_bytes(auth))).strip()
self.header('Authorization', 'Basic %s' % auth)
self.push(CRLF)
self.push(CRLF)
def chunked_size(self):
line = self.buffer
if not line:
return
try:
crlf = as_bytes(CRLF) if PY3 and isinstance(line, bytes) else CRLF
data = re.split(re.escape(crlf), line)
chunk_size = int(data[0], 16)
except ValueError:
return
if chunk_size == 0:
self.part = self.trailer
else:
self.set_terminator(chunk_size)
self.part = self.chunked_body
self.length += chunk_size
def push (self, data):
data = as_bytes(data)
self.producer_fifo.push(simple_producer(data))
self.initiate_send()
def continue_request(self, data, request):
logger = self.supervisord.options.logger
try:
try:
# on 2.x, the Expat parser doesn't like Unicode which actually
# contains non-ASCII characters. It's a bit of a kludge to
# do it conditionally here, but it's down to how underlying
# libs behave
if not PY3:
data = data.encode('ascii', 'xmlcharrefreplace')
params, method = self.loads(data)
except:
logger.error(
'XML-RPC request data %r is invalid: unmarshallable' %
(data, ))
request.error(400)
return
# no <methodName> in the request or name is an empty string
if not method:
logger.error(
'XML-RPC request data %r is invalid: no method name' %
(data, ))
request.error(400)
return
# we allow xml-rpc clients that do not send empty <params>
# when there are no parameters for the method call
if params is None:
params = ()
try:
logger.trace('XML-RPC method called: %s()' % method)
value = self.call(method, params)
logger.trace('XML-RPC method %s() returned successfully' %
method)
except RPCError as err:
# turn RPCError reported by method into a Fault instance
value = xmlrpclib.Fault(err.code, err.text)
logger.trace('XML-RPC method %s() returned fault: [%d] %s' %
(method, err.code, err.text))
if isinstance(value, types.FunctionType):
# returning a function from an RPC method implies that
# this needs to be a deferred response (it needs to block).
pushproducer = request.channel.push_with_producer
pushproducer(DeferredXMLRPCResponse(request, value))
else:
# if we get anything but a function, it implies that this
# response doesn't need to be deferred, we can service it
# right away.
body = as_bytes(xmlrpc_marshal(value))
request['Content-Type'] = 'text/xml'
request['Content-Length'] = len(body)
request.push(body)
request.done()
except:
tb = traceback.format_exc()
logger.critical(
"Handling XML-RPC request with data %r raised an unexpected "
"exception: %s" % (data, tb))
# internal error, report as HTTP server error
request.error(500)
import datetime, time
def make_datetime(text):
return datetime.datetime(*time.strptime(text, "%Y%m%dT%H:%M:%S")[:6])
unmarshallers = {
"int": lambda x: int(x.text),
"i4": lambda x: int(x.text),
"boolean": lambda x: x.text == "1",
"string": lambda x: x.text or "",
"double": lambda x: float(x.text),
"dateTime.iso8601": lambda x: make_datetime(x.text),
"array": lambda x: x[0].text,
"data": lambda x: [v.text for v in x],
"struct": lambda x: dict([(k.text or "", v.text) for k, v in x]),
"base64": lambda x: as_string(decodestring(as_bytes(x.text or ""))),
"value": lambda x: x[0].text,
"param": lambda x: x[0].text,
}
def loads(data):
params = method = None
for action, elem in iterparse(StringIO(data)):
unmarshall = unmarshallers.get(elem.tag)
if unmarshall:
data = unmarshall(elem)
elem.clear()
elem.text = data
elif elem.tag == "methodName":
method = elem.text
elif elem.tag == "params":
class supervisor_xmlrpc_handler(xmlrpc_handler):
path = '/RPC2'
IDENT = 'Supervisor XML-RPC Handler'
unmarshallers = {
"int": lambda x: int(x.text),
"i4": lambda x: int(x.text),
"boolean": lambda x: x.text == "1",
"string": lambda x: x.text or "",
"double": lambda x: float(x.text),
"dateTime.iso8601": lambda x: make_datetime(x.text),
"array": lambda x: x[0].text,
"data": lambda x: [v.text for v in x],
"struct": lambda x: dict([(k.text or "", v.text) for k, v in x]),
"base64": lambda x: as_string(decodestring(as_bytes(x.text or ""))),
"param": lambda x: x[0].text,
}
def __init__(self, supervisord, subinterfaces):
self.rpcinterface = RootRPCInterface(subinterfaces)
self.supervisord = supervisord
def loads(self, data):
params = method = None
for action, elem in iterparse(StringIO(data)):
unmarshall = self.unmarshallers.get(elem.tag)
if unmarshall:
data = unmarshall(elem)
elem.clear()
elem.text = data
elif elem.tag == "value":
try:
data = elem[0].text
except IndexError:
data = elem.text or ""
elem.clear()
elem.text = data
elif elem.tag == "methodName":
method = elem.text
elif elem.tag == "params":
params = tuple([v.text for v in elem])
return params, method
def match(self, request):
return request.uri.startswith(self.path)
def continue_request(self, data, request):
logger = self.supervisord.options.logger
try:
try:
params, method = self.loads(data)
except:
logger.error(
'XML-RPC request data %r is invalid: unmarshallable' %
(data, ))
request.error(400)
return
# no <methodName> in the request or name is an empty string
if not method:
logger.error(
'XML-RPC request data %r is invalid: no method name' %
(data, ))
request.error(400)
return
# we allow xml-rpc clients that do not send empty <params>
# when there are no parameters for the method call
if params is None:
params = ()
try:
logger.trace('XML-RPC method called: %s()' % method)
value = self.call(method, params)
logger.trace('XML-RPC method %s() returned successfully' %
method)
except RPCError as err:
# turn RPCError reported by method into a Fault instance
value = xmlrpclib.Fault(err.code, err.text)
logger.trace('XML-RPC method %s() returned fault: [%d] %s' %
(method, err.code, err.text))
if isinstance(value, types.FunctionType):
# returning a function from an RPC method implies that
# this needs to be a deferred response (it needs to block).
pushproducer = request.channel.push_with_producer
pushproducer(DeferredXMLRPCResponse(request, value))
else:
# if we get anything but a function, it implies that this
# response doesn't need to be deferred, we can service it
# right away.
body = xmlrpc_marshal(value)
request['Content-Type'] = 'text/xml'
request['Content-Length'] = len(body)
request.push(body)
request.done()
except:
tb = traceback.format_exc()
logger.critical(
"Handling XML-RPC request with data %r raised an unexpected "
"exception: %s" % (data, tb))
# internal error, report as HTTP server error
request.error(500)
def call(self, method, params):
return traverse(self.rpcinterface, method, params)
def send(self, s):
return os.write(self.fd, as_bytes(s))
def test_authorize_gooduser_goodpassword_sha(self):
password = '******' + sha1(as_bytes('password')).hexdigest()
authorizer = self._makeOne({'foo': password})
self.assertTrue(authorizer.authorize(('foo', 'password')))
def write(self, msg):
if self.error:
error = self.error
self.error = None
raise error
self.written += as_bytes(msg)
def push (self, data):
data = as_bytes(data)
# 推入一个给定 数据的生产者
self.producer_fifo.push(simple_producer(data))
# 发送
self.initiate_send()