class Task:
"""Represents the task of serving a single HTTP Request.
The httpy._zope.server machinery instantiates this once for each HTTP
request, and calls service in its own thread.
"""
implements(ITask)
def __init__(self, channel, request):
"""Takes an IServerChannel (httpy._zope) and an IRequest (httpy).
"""
try:
# Set these early in case we fail.
self.channel = channel
self.server_config = self.channel.server.config
self.dev_mode = os.environ['HTTPY_MODE'] == 'development'
# This is where we are likely to fail.
self.request = Request(request)
self.app = self.find_app()
if int(os.environ.get('HTTPY_VERBOSITY', 0)) >= 99:
request_lines = request.raw.splitlines()
raw_request = os.linesep.join(request_lines)
log(99, raw_request)
except:
self.fail()
def find_app(self):
"""Find an app with which to serve the requested URI.
"""
app = None
for _app in self.server_config.apps:
if self.request.path.startswith(_app.uri_root):
app = _app
break
if app is None:
# This catches, e.g., ../../../../../../../../../etc/master.passwd
raise Exception("Unable to find an application to serve " +
"%s." % self.request.path)
log(98, "Using %s for this request" % app)
return app
# ITask contracts
# ===============
# service, cancel, defer
def service(self):
try:
self.respond()
except Response, response:
self.deliver(response)
except:
class SyslogLogger(m_syslog.syslog_client):
"""syslog is a line-oriented log protocol - this class would be
appropriate for FTP or HTTP logs, but not for dumping stderr
to.
TODO: a simple safety wrapper that will ensure that the line
sent to syslog is reasonable.
TODO: async version of syslog_client: now, log entries use
blocking send()
"""
implements(IMessageLogger)
svc_name = 'httpy._zope'
pid_str = str(os.getpid())
def __init__(self, address, facility='user'):
m_syslog.syslog_client.__init__(self, address)
self.facility = m_syslog.facility_names[facility]
self.address = address
def __repr__(self):
return '<syslog logger address=%s>' % (repr(self.address))
def logMessage(self, message):
'See IMessageLogger'
m_syslog.syslog_client.log(self,
'%s[%s]: %s' %
(self.svc_name, self.pid_str, message),
facility=self.facility,
priority=m_syslog.LOG_INFO)
class Responder:
"""This default responder infers a site from your filesystem.
"""
implements(IBaseResponder)
root = ''
def __init__(self, root=None):
# Set paths.
# ==========
root = root or os.getcwd()
if not os.path.isdir(root):
raise ValueError("root '%s' does not point to a directory" % root)
self.root = os.path.realpath(root)
self.__ = os.path.join(self.root, '__')
if not os.path.exists(self.__):
self.__ = None
# Find findables.
# ===============
self.framework = self.get_framework()
self.responders = self.get_responders()
# Contracts
# =========
def respond(self, request):
"""Given a Request, return an IResponse.
This is where we insert our framework hooks.
"""
# Inbound hooks
# =============
if hasattr(self.framework, 'get_responder'):
responder = self.framework.get_responder(request)
else:
responder = self.get_responder(request)
if hasattr(self.framework, 'wrap_request'):
request = self.framework.wrap_request(responder, request)
# Get a response
# ==============
try:
response = responder.respond(request)
except Response, response:
response = response
except:
class ResolvingLogger(object):
"""Feed (ip, message) combinations into this logger to get a
resolved hostname in front of the message. The message will not
be logged until the PTR request finishes (or fails)."""
implements(IRequestLogger)
def __init__(self, resolver, logger):
self.resolver = resolver
# logger is an IMessageLogger
self.logger = logger
class logger_thunk(object):
def __init__(self, message, logger):
self.message = message
self.logger = logger
def __call__(self, host, ttl, answer):
if not answer:
answer = host
self.logger.logMessage('%s%s' % (answer, self.message))
def logRequest(self, ip, message):
'See IRequestLogger'
self.resolver.resolve_ptr(ip, self.logger_thunk(message, self.logger))
class Application:
"""This is httpy's default request respondor.
"""
implements(IApplication)
def __init__(self):
"""Takes a ApplicationConfig object.
"""
self.dev_mode = os.environ.get("HTTPY_MODE") == 'development'
from httpy import utils # dodge circular import
self.uri_to_fs = utils.uri_to_fs
def __repr__(self):
return "<default httpy app>"
__str__ = __repr__
def respond(self, request):
"""Given an httpy.Request, raise an httpy.Response.
"""
self.request = request
fs_path = self.uri_to_fs(self.site_root,
self.fs_root,
self.uri_root,
request.path,
defaults=('index.html', 'index.htm'))
self.serve_static(fs_path)
def serve_static(self, fs_path):
"""Given a filesystem path to a static resource, serve it.
"""
# Get basic info from the filesystem and start building a response.
# =================================================================
mtime = os.stat(fs_path)[stat.ST_MTIME]
content_type = mimetypes.guess_type(fs_path)[0] or 'text/plain'
response = Response(200)
# Support 304s, but only in deployment mode.
# ==========================================
if not self.dev_mode:
ims = self.request.message.get('If-Modified-Since')
if ims:
mod_since = rfc822.parsedate(ims)
last_modified = time.gmtime(mtime)
if last_modified[:6] <= mod_since[:6]:
response.code = 304
# Finish building the response and raise it.
# ========================================
response.headers['Last-Modified'] = rfc822.formatdate(mtime)
response.headers['Content-Type'] = content_type
if response.code != 304:
response.body = file(fs_path, 'rb').read()
raise response
class UnresolvingLogger(object):
"""Just in case you don't want to resolve"""
implements(IRequestLogger)
def __init__(self, logger):
self.logger = logger
def logRequest(self, ip, message):
'See IRequestLogger'
self.logger.logMessage('%s%s' % (ip, message))
class SleepingTask(object):
implements(ITask)
def service(self):
sleep(0.2)
def cancel(self):
pass
def defer(self):
pass
class Transaction:
"""This is httpy's default request processor.
"""
implements(ITransaction)
def __init__(self, config):
"""Takes a TransactionConfig object.
"""
self.config = config
def process(self, request):
"""Given an httpy.Request, raise an httpy.Response.
"""
self.request = request
fs_path = uri_to_fs(config=self.config,
resource_uri_path=request.path,
defaults=('index.html', 'index.htm'))
self.serve_static(fs_path)
def serve_static(self, fs_path):
"""Given a filesystem path to a static resource, serve it.
"""
# Get basic info from the filesystem and start building a response.
# =================================================================
mtime = os.stat(fs_path)[stat.ST_MTIME]
content_type = mimetypes.guess_type(fs_path)[0] or 'text/plain'
response = Response(200)
# Support 304s, but only in deployment mode.
# ==========================================
if self.config.mode == 'deployment':
ims = self.request.message.get('If-Modified-Since')
if ims:
mod_since = rfc822.parsedate(ims)
last_modified = time.gmtime(mtime)
if last_modified[:6] <= mod_since[:6]:
response.code = 304
# Finish building the response and raise it.
# ========================================
response.headers['Last-Modified'] = rfc822.formatdate(mtime)
response.headers['Content-Type'] = content_type
if response.code != 304:
response.body = file(fs_path, 'rb').read()
raise response
class FileLogger(object):
"""Simple File Logger
"""
implements(IMessageLogger)
def __init__(self, file, flush=1, mode='a'):
"""pass this either a path or a file object."""
if type(file) is StringType:
if (file == '-'):
import sys
self.file = sys.stdout
else:
self.file = open(file, mode)
else:
self.file = file
self.do_flush = flush
def __repr__(self):
return '<file logger: %s>' % self.file
def write(self, data):
self.file.write(data)
self.maybe_flush()
def writeline(self, line):
self.file.writeline(line)
self.maybe_flush()
def writelines(self, lines):
self.file.writelines(lines)
self.maybe_flush()
def maybe_flush(self):
if self.do_flush:
self.file.flush()
def flush(self):
self.file.flush()
def softspace(self, *args):
pass
def logMessage(self, message):
'See IMessageLogger'
if message[-1] not in ('\r', '\n'):
self.write(message + '\n')
else:
self.write(message)
class TailLogger(object):
"""Keep track of the last <size> log messages"""
implements(IMessageLogger)
def __init__(self, logger, size=500):
self.size = size
self.logger = logger
self.messages = []
def logMessage(self, message):
'See IMessageLogger'
self.messages.append(strip_eol(message))
if len(self.messages) > self.size:
del self.messages[0]
self.logger.logMessage(message)
class LineTask(object):
"""This is a generic task that can be used with command line
protocols to handle commands in a separate thread.
"""
implements(ITask)
def __init__(self, channel, command, m_name):
self.channel = channel
self.m_name = m_name
self.args = command.args
self.close_on_finish = 0
def service(self):
"""Called to execute the task.
"""
try:
try:
self.start()
getattr(self.channel, self.m_name)(self.args)
self.finish()
except socket.error:
self.close_on_finish = 1
if self.channel.adj.log_socket_errors:
raise
except:
self.channel.exception()
finally:
if self.close_on_finish:
self.channel.close_when_done()
def cancel(self):
'See ITask'
self.channel.close_when_done()
def defer(self):
'See ITask'
pass
def start(self):
now = time.time()
self.start_time = now
def finish(self):
hit_log = self.channel.server.hit_log
if hit_log is not None:
hit_log.log(self)
class LineCommandParser(object):
"""Line Command parser. Arguments are left alone for now."""
implements(IStreamConsumer)
# See IStreamConsumer
completed = 0
inbuf = ''
cmd = ''
args = ''
empty = 0
max_line_length = 1024 # Not a hard limit
def __init__(self, adj):
"""
adj is an Adjustments object.
"""
self.adj = adj
def received(self, data):
'See IStreamConsumer'
if self.completed:
return 0 # Can't consume any more.
pos = data.find('\n')
datalen = len(data)
if pos < 0:
self.inbuf = self.inbuf + data
if len(self.inbuf) > self.max_line_length:
# Don't accept any more.
self.completed = 1
return datalen
else:
# Line finished.
s = data[:pos + 1]
self.inbuf = self.inbuf + s
self.completed = 1
line = self.inbuf.strip()
self.parseLine(line)
return len(s)
def parseLine(self, line):
parts = line.split(' ', 1)
if len(parts) == 2:
self.cmd, self.args = parts
else:
self.cmd = parts[0]
class PythonLogger(object):
"""Proxy for Python's logging module"""
implements(IMessageLogger)
def __init__(self, name=None, level=logging.INFO):
self.name = name
self.level = level
self.logger = logging.getLogger(name)
def __repr__(self):
return '<python logger: %s %s>' % (self.name,
logging.getLevelName(self.level))
def logMessage(self, message):
"""See IMessageLogger"""
self.logger.log(self.level, message.rstrip())
class Task:
"""Represents the task of serving a single HTTP Request.
The httpy._zope.server machinery instantiates this once for each HTTP
request, and calls service in its own thread.
"""
implements(ITask)
def __init__(self, channel, request):
"""Takes an IServerChannel (httpy._zope) and an IRequest (httpy).
"""
self.channel = channel
self.server = self.channel.server
self.responder = self.channel.server.responder
self.request = request
self.deliver = make_deliver(self.make_request, self.responder,
self.channel)
if 0: # turned off cause it eats CPU even when not printed
request_lines = request.raw.splitlines()
raw_request = os.linesep.join(request_lines)
logger.debug(raw_request)
def make_request(self, request):
return Request(request)
# ITask contracts
# ===============
# service, cancel, defer
def service(self):
self.deliver(self.request)
self.channel.close_when_done()
def cancel(self):
self.channel.close_when_done()
def defer(self):
pass
class DemoFileSystemAccess(object):
__doc__ = IFileSystemAccess.__doc__
implements(IFileSystemAccess)
def __init__(self, files, users):
self.files = files
self.users = users
def authenticate(self, credentials):
"See httpy._zope.server.interfaces.ftp.IFileSystemAccess"
user, password = credentials
if user != 'anonymous':
if self.users.get(user) != password:
raise Unauthorized
return user
def open(self, credentials):
"See httpy._zope.server.interfaces.ftp.IFileSystemAccess"
user = self.authenticate(credentials)
return DemoFileSystem(self.files, user)
class Task:
"""Represents the task of serving a single HTTP Request.
The httpy._zope.server machinery instantiates this once for each HTTP
request, and calls service in its own thread.
"""
implements(ITask)
def __init__(self, channel, request):
"""Takes an IServerChannel (httpy._zope) and an IRequest (httpy).
"""
self.channel = channel
self.server = self.channel.server
self.responder = self.channel.server.responder
self.request = request
self.deliver = Deliver( Request
, self.responder
, self.channel
)
# ITask contracts
# ===============
# service, cancel, defer
def service(self):
self.deliver(self.request)
self.channel.close_when_done()
def cancel(self):
self.channel.close_when_done()
def defer(self):
pass
class SocketLogger(asynchat.async_chat):
"""Log to a stream socket, asynchronously."""
implements(IMessageLogger)
def __init__(self, address):
if type(address) == type(''):
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
else:
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect(address)
self.address = address
def __repr__(self):
return '<socket logger: address=%s>' % (self.address)
def logMessage(self, message):
'See IMessageLogger'
if message[-2:] != '\r\n':
self.socket.push(message + '\r\n')
else:
self.socket.push(message)
class DemoFileSystem(object):
__doc__ = IFileSystem.__doc__
implements(IFileSystem)
File = File
Directory = Directory
def __init__(self, files, user=''):
self.files = files
self.user = user
def get(self, path, default=None):
while path.startswith('/'):
path = path[1:]
d = self.files
if path:
for name in path.split('/'):
if d.type is not 'd':
return default
if not d.accessable(self.user):
raise Unauthorized
d = d.get(name)
if d is None:
break
return d
def getany(self, path):
d = self.get(path)
if d is None:
raise OSError("No such file or directory:", path)
return d
def getdir(self, path):
d = self.getany(path)
if d.type != 'd':
raise OSError("Not a directory:", path)
return d
def getfile(self, path):
d = self.getany(path)
if d.type != 'f':
raise OSError("Not a file:", path)
return d
def getwdir(self, path):
d = self.getdir(path)
if not d.accessable(self.user, write):
raise OSError("Permission denied")
return d
def type(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = self.get(path)
return getattr(f, 'type', None)
def names(self, path, filter=None):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = list(self.getdir(path))
if filter is not None:
f = [name for name in f if filter(name)]
return f
def _lsinfo(self, name, file):
info = {
'type': file.type,
'name': name,
'group_read': file.accessable(self.user, read),
'group_write': file.accessable(self.user, write),
}
if file.type == 'f':
info['size'] = len(file.data)
if file.modified is not None:
info['mtime'] = file.modified
return info
def ls(self, path, filter=None):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = self.getdir(path)
if filter is None:
return [self._lsinfo(name, f.files[name])
for name in f
]
return [self._lsinfo(name, f.files[name])
for name in f
if filter(name)]
def readfile(self, path, outstream, start=0, end=None):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = self.getfile(path)
data = f.data
if end is not None:
data = data[:end]
if start:
data = data[start:]
outstream.write(data)
def lsinfo(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = self.getany(path)
return self._lsinfo(posixpath.split(path)[1], f)
def mtime(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = self.getany(path)
return f.modified
def size(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
f = self.getany(path)
return len(getattr(f, 'data', ''))
def mkdir(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
path, name = posixpath.split(path)
d = self.getwdir(path)
if name in d.files:
raise OSError("Already exists:", name)
newdir = self.Directory()
newdir.grant(self.user, read | write)
d.files[name] = newdir
def remove(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
path, name = posixpath.split(path)
d = self.getwdir(path)
if name not in d.files:
raise OSError("Not exists:", name)
f = d.files[name]
if f.type == 'd':
raise OSError('Is a directory:', name)
del d.files[name]
def rmdir(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
path, name = posixpath.split(path)
d = self.getwdir(path)
if name not in d.files:
raise OSError("Not exists:", name)
f = d.files[name]
if f.type != 'd':
raise OSError('Is not a directory:', name)
del d.files[name]
def rename(self, old, new):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
oldpath, oldname = posixpath.split(old)
newpath, newname = posixpath.split(new)
olddir = self.getwdir(oldpath)
newdir = self.getwdir(newpath)
if oldname not in olddir.files:
raise OSError("Not exists:", oldname)
if newname in newdir.files:
raise OSError("Already exists:", newname)
newdir.files[newname] = olddir.files[oldname]
del olddir.files[oldname]
def writefile(self, path, instream, start=None, end=None, append=False):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
path, name = posixpath.split(path)
d = self.getdir(path)
f = d.files.get(name)
if f is None:
d = self.getwdir(path)
f = d.files[name] = self.File()
f.grant(self.user, read | write)
elif f.type != 'f':
raise OSError("Can't overwrite a directory")
if not f.accessable(self.user, write):
raise OSError("Permission denied")
if append:
f.data += instream.read()
else:
if start:
if start < 0:
raise ValueError("Negative starting file position")
prefix = f.data[:start]
if len(prefix) < start:
prefix += '\0' * (start - len(prefix))
else:
prefix = ''
start=0
if end:
if end < 0:
raise ValueError("Negative ending file position")
l = end - start
newdata = instream.read(l)
f.data = prefix+newdata+f.data[start+len(newdata):]
else:
f.data = prefix + instream.read()
def writable(self, path):
"See httpy._zope.server.interfaces.ftp.IFileSystem"
path, name = posixpath.split(path)
try:
d = self.getdir(path)
except OSError:
return False
if name not in d:
return d.accessable(self.user, write)
f = d[name]
return f.type == 'f' and f.accessable(self.user, write)
class ServerChannelBase(DualModeChannel, object):
"""Base class for a high-performance, mixed-mode server-side channel."""
implements(IServerChannel, ITask)
# See httpy._zope.server.interfaces.IServerChannel
parser_class = None # Subclasses must provide a parser class
task_class = None # ... and a task class.
active_channels = {} # Class-specific channel tracker
next_channel_cleanup = [0] # Class-specific cleanup time
proto_request = None # A request parser instance
last_activity = 0 # Time of last activity
tasks = None # List of channel-related tasks to execute
running_tasks = False # True when another thread is running tasks
#
# ASYNCHRONOUS METHODS (including __init__)
#
def __init__(self, server, conn, addr, adj=None):
"""See async.dispatcher"""
DualModeChannel.__init__(self, conn, addr, adj)
self.server = server
self.last_activity = t = self.creation_time
self.check_maintenance(t)
def add_channel(self, map=None):
"""See async.dispatcher
This hook keeps track of opened channels.
"""
DualModeChannel.add_channel(self, map)
self.__class__.active_channels[self._fileno] = self
def del_channel(self, map=None):
"""See async.dispatcher
This hook keeps track of closed channels.
"""
DualModeChannel.del_channel(self, map)
ac = self.__class__.active_channels
fd = self._fileno
if fd in ac:
del ac[fd]
def check_maintenance(self, now):
"""See async.dispatcher
Performs maintenance if necessary.
"""
ncc = self.__class__.next_channel_cleanup
if now < ncc[0]:
return
ncc[0] = now + self.adj.cleanup_interval
self.maintenance()
def maintenance(self):
"""See async.dispatcher
Kills off dead connections.
"""
self.kill_zombies()
def kill_zombies(self):
"""See async.dispatcher
Closes connections that have not had any activity in a while.
The timeout is configured through adj.channel_timeout (seconds).
"""
now = time.time()
cutoff = now - self.adj.channel_timeout
for channel in self.active_channels.values():
if (channel is not self and not channel.running_tasks
and channel.last_activity < cutoff):
channel.close()
def received(self, data):
"""See async.dispatcher
Receives input asynchronously and send requests to
handle_request().
"""
preq = self.proto_request
while data:
if preq is None:
preq = self.parser_class(self.adj)
n = preq.received(data)
if preq.completed:
# The request is ready to use.
self.proto_request = None
if not preq.empty:
self.handle_request(preq)
preq = None
else:
self.proto_request = preq
if n >= len(data):
break
data = data[n:]
def handle_request(self, req):
"""Creates and queues a task for processing a request.
Subclasses may override this method to handle some requests
immediately in the main async thread.
"""
task = self.task_class(self, req)
self.queue_task(task)
def handle_error(self):
"""See async.dispatcher
Handles program errors (not communication errors)
"""
t, v = sys.exc_info()[:2]
if t is SystemExit or t is KeyboardInterrupt:
raise t, v
asyncore.dispatcher.handle_error(self)
def handle_comm_error(self):
"""See async.dispatcher
Handles communication errors (not program errors)
"""
if self.adj.log_socket_errors:
self.handle_error()
else:
# Ignore socket errors.
self.close()
#
# BOTH MODES
#
def queue_task(self, task):
"""Queue a channel-related task to be executed in another thread."""
start = False
task_lock.acquire()
try:
if self.tasks is None:
self.tasks = []
self.tasks.append(task)
if not self.running_tasks:
self.running_tasks = True
start = True
finally:
task_lock.release()
if start:
self.set_sync()
self.server.addTask(self)
#
# ITask implementation. Delegates to the queued tasks.
#
def service(self):
"""Execute all pending tasks"""
while True:
task = None
task_lock.acquire()
try:
if self.tasks:
task = self.tasks.pop(0)
else:
# No more tasks
self.running_tasks = False
self.set_async()
break
finally:
task_lock.release()
try:
task.service()
except:
# propagate the exception, but keep executing tasks
self.server.addTask(self)
raise
def cancel(self):
"""Cancels all pending tasks"""
task_lock.acquire()
try:
if self.tasks:
old = self.tasks[:]
else:
old = []
self.tasks = []
self.running_tasks = False
finally:
task_lock.release()
try:
for task in old:
task.cancel()
finally:
self.set_async()
def defer(self):
pass
class ServerBase(asyncore.dispatcher, object):
"""Async. server base for launching derivatives of ServerChannelBase."""
implements(IServer)
# See httpy._zope.server.interfaces.IServer
channel_class = None # Override with a channel class.
SERVER_IDENT = 'httpy._zope.server.serverbase' # Override.
def __init__(self,
ip,
port,
task_dispatcher=None,
adj=None,
start=1,
hit_log=None,
verbose=0):
if adj is None:
adj = default_adj
self.adj = adj
asyncore.dispatcher.__init__(self)
self.port = port
self.task_dispatcher = task_dispatcher
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((ip, port))
self.verbose = verbose
self.hit_log = hit_log
self.logger = logging.getLogger(self.__class__.__name__)
self.server_name = self.computeServerName(ip)
if start:
self.accept_connections()
def log(self, message):
"""See httpy._zope.server.interfaces.IDispatcherLogging"""
# Override asyncore's default log()
self.logger.info(message)
level_mapping = {
'info': logging.INFO,
'error': logging.ERROR,
'warning': logging.WARN,
}
def log_info(self, message, type='info'):
"""See httpy._zope.server.interfaces.IDispatcherLogging"""
self.logger.log(self.level_mapping.get(type, logging.INFO), message)
def computeServerName(self, ip=''):
"""Given an IP, try to determine the server name."""
if ip:
server_name = str(ip)
else:
server_name = str(socket.gethostname())
# Convert to a host name if necessary.
is_hostname = 0
for c in server_name:
if c != '.' and not c.isdigit():
is_hostname = 1
break
if not is_hostname:
if self.verbose:
self.log_info('Computing hostname', 'info')
try:
server_name = socket.gethostbyaddr(server_name)[0]
except socket.error:
if self.verbose:
self.log_info('Cannot do reverse lookup', 'info')
return server_name
def accept_connections(self):
self.accepting = 1
self.socket.listen(self.adj.backlog) # Circumvent asyncore's NT limit
if self.verbose:
self.log_info('%s started.\n'
'\tHostname: %s\n\tPort: %d' %
(self.SERVER_IDENT, self.server_name, self.port))
def addTask(self, task):
"""See httpy._zope.server.interfaces.ITaskDispatcher"""
td = self.task_dispatcher
if td is not None:
td.addTask(task)
else:
task.service()
def readable(self):
"""See httpy._zope.server.interfaces.IDispatcher"""
return (self.accepting
and len(asyncore.socket_map) < self.adj.connection_limit)
def writable(self):
"""See httpy._zope.server.interfaces.IDispatcher"""
return 0
def handle_read(self):
"""See httpy._zope.server.interfaces.IDispatcherEventHandler"""
pass
def handle_connect(self):
"""See httpy._zope.server.interfaces.IDispatcherEventHandler"""
pass
def handle_accept(self):
"""See httpy._zope.server.interfaces.IDispatcherEventHandler"""
try:
v = self.accept()
if v is None:
return
conn, addr = v
except socket.error:
# Linux: On rare occasions we get a bogus socket back from
# accept. socketmodule.c:makesockaddr complains that the
# address family is unknown. We don't want the whole server
# to shut down because of this.
if self.adj.log_socket_errors:
self.log_info('warning: server accept() threw an exception',
'warning')
return
for (level, optname, value) in self.adj.socket_options:
conn.setsockopt(level, optname, value)
self.channel_class(self, conn, addr, self.adj)
class Foo:
implements(IFoo)
def foo(self):
pass
class ChunkedReceiver(object):
implements(IStreamConsumer)
chunk_remainder = 0
control_line = ''
all_chunks_received = 0
trailer = ''
completed = 0
# max_control_line = 1024
# max_trailer = 65536
def __init__(self, buf):
self.buf = buf
def received(self, s):
# Returns the number of bytes consumed.
if self.completed:
return 0
orig_size = len(s)
while s:
rm = self.chunk_remainder
if rm > 0:
# Receive the remainder of a chunk.
to_write = s[:rm]
self.buf.append(to_write)
written = len(to_write)
s = s[written:]
self.chunk_remainder -= written
elif not self.all_chunks_received:
# Receive a control line.
s = self.control_line + s
pos = s.find('\n')
if pos < 0:
# Control line not finished.
self.control_line = s
s = ''
else:
# Control line finished.
line = s[:pos]
s = s[pos + 1:]
self.control_line = ''
line = line.strip()
if line:
# Begin a new chunk.
semi = line.find(';')
if semi >= 0:
# discard extension info.
line = line[:semi]
sz = int(line.strip(), 16) # hexadecimal
if sz > 0:
# Start a new chunk.
self.chunk_remainder = sz
else:
# Finished chunks.
self.all_chunks_received = 1
# else expect a control line.
else:
# Receive the trailer.
trailer = self.trailer + s
if trailer.startswith('\r\n'):
# No trailer.
self.completed = 1
return orig_size - (len(trailer) - 2)
elif trailer.startswith('\n'):
# No trailer.
self.completed = 1
return orig_size - (len(trailer) - 1)
pos = find_double_newline(trailer)
if pos < 0:
# Trailer not finished.
self.trailer = trailer
s = ''
else:
# Finished the trailer.
self.completed = 1
self.trailer = trailer[:pos]
return orig_size - (len(trailer) - pos)
return orig_size
def getfile(self):
return self.buf.getfile()
class Application:
"""This is httpy's default request respondor.
"""
implements(IApplication)
name = 'default httpy app'
def __init__(self):
from httpy import utils # dodge circular import -- still a problem?
self.uri_to_fs = utils.uri_to_fs
def __repr__(self):
return "<%s>" % self.name
__str__ = __repr__
def respond(self, request):
"""Given an httpy.Request, raise an httpy.Response.
"""
fs_path = self.uri_to_fs(self.site_root,
self.fs_root,
self.uri_root,
request.path,
defaults=('index.html', 'index.htm'))
ims = request.message.get('If-Modified-Since', '')
self.serve_static(fs_path, ims)
def serve_static(self, fs_path, ims):
"""Given a filesystem path to a static resource, serve it.
This is factored out for easier reuse.
"""
# Get basic info from the filesystem and start building a response.
# =================================================================
mtime = os.stat(fs_path)[stat.ST_MTIME]
content_type = mimetypes.guess_type(fs_path)[0] or 'text/plain'
response = Response(200)
# Support 304s, but only in deployment mode.
# ==========================================
if self.deploy_mode:
if ims:
mod_since = rfc822.parsedate(ims)
last_modified = time.gmtime(mtime)
if last_modified[:6] <= mod_since[:6]:
response.code = 304
# Finish building the response and raise it.
# ========================================
response.headers['Last-Modified'] = rfc822.formatdate(mtime)
response.headers['Content-Type'] = content_type
if response.code != 304:
response.body = file(fs_path, 'rb').read()
raise response
def close(self):
logger.debug("closing DefaultApp")
class Task:
"""Represents the task of serving a single HTTP Request.
The httpy._zope.server machinery instantiates this once for each HTTP
request, and calls service in its own thread.
"""
implements(ITask)
def __init__(self, channel, request):
"""Takes an IServerChannel (httpy._zope) and an IRequest (httpy).
"""
try:
# Set these early in case we fail.
self.channel = channel
self.server = self.channel.server
self.framework = self.channel.server.framework
# This is where we are likely to fail.
self.request = Request(request)
self.app = self.get_app()
if 0: # turned off cause it eats CPU even when not printed
request_lines = request.raw.splitlines()
raw_request = os.linesep.join(request_lines)
logger.debug(raw_request)
except:
self.fail()
def get_app(self):
"""Get an application with which to serve the requested URI.
"""
app = None
for _app in self.server.apps:
if self.request.path.startswith(_app.uri_root):
app = _app
break
if app is None:
# This catches, e.g., ../../../../../../../../../etc/master.passwd
raise StandardError("Unable to find an application to serve " +
"%s." % self.request.path)
logger.debug("Using %s for this request." % app)
return app
# ITask contracts
# ===============
# service, cancel, defer
def service(self):
try:
self.respond()
except ResponseFlag, response:
try:
# outbound framework hook
response = self.framework.unwrap_response(self.app, response)
self.deliver(response)
except:
self.fail()
except:
class CGIRequest:
"""Reconstruct an httpy.Request from the environment and stdin.
"""
implements(IRequest)
def __init__(self, environ, stdin):
"""Takes a dictionary and a buffer.
"""
# Request-Line
# ============
method = environ['REQUEST_METHOD']
path = environ.get('PATH_INFO', '/')
query = environ.get('QUERY_STRING', '')
if query:
query = '?' + query
raw_uri = environ.get('SCRIPT_NAME', '') + path + query
uri = urlparse.urlparse(raw_uri)
keys = ( 'scheme'
, 'netloc'
, 'path'
, 'parameters'
, 'query'
, 'fragment'
)
_uri = {}
for i in range(len(uri)):
k = keys[i]
v = uri[i]
_uri[k] = v
uri = _uri
http_version = environ['SERVER_PROTOCOL']
raw_line = "%s %s %s\r\n" % (method, raw_uri, http_version)
# Headers
# =======
headers = []
WANT = ('content_type', 'content_length')
for k, v in environ.iteritems():
want = False
k = k.lower()
if k.startswith('http_'):
k = k[5:]
want = True
if want or (k in WANT):
as_string = "%s: %s" % (k.replace('_','-'), v)
headers.append(as_string)
raw_headers = '\r\n'.join(headers)
raw_headers += '\r\n'
headers = message_from_string(raw_headers)
# Body
# ====
raw_body = stdin.read()
# Save the API we want.
# =====================
raw = raw_line + raw_headers + raw_body
self.raw = raw
self.raw_line = raw_line
self.raw_headers = raw_headers
self.raw_body = raw_body
self.method = method
self.uri = uri
self.path = path
self.headers = headers
class ZopeRequest:
"""An implementation of IRequest that works with httpy._zope.server.
"""
implements(IStreamConsumer, IRequest)
raw = ''
raw_line = ''
raw_headers = ''
raw_body = ''
method = ''
uri = {}
path = ''
headers = Message()
def __init__(self, adj=None):
"""Takes an Adjustments object.
"""
if adj is None:
adj = default_adj
self.adj = adj
self._receiver = None
self._tmp = ''
self._raw = []
# initialize mutable defaults
self.uri = {}
self.headers = Message()
# Fulfill our IStreamConsumer contracts.
# ======================================
def received(self, block):
"""This takes a block of an incoming HTTP stream.
We store anything leftover from the last block in raw string form in
self._tmp.
"""
self._raw.append(block)
try:
if not self.raw_line:
self.get_line(block)
elif not self.raw_headers:
self.get_headers(block)
elif not self.raw_body:
self.get_body(block)
except Complete:
self.raw = ''.join(self._raw)
self.completed = True
return len(block)
completed = False
empty = False
# Request-Line
# ============
def get_line(self, block):
"""
"""
block = self._tmp + block
if '\r' in block or '\n' in block:
lines = block.splitlines(True)
i = 0
for line in lines:
if not line.strip(): #could be a blank line (like from IE)
pass
elif (line.endswith('\r')
or line.endswith('\n')): #could be the whole thing
self.raw_line = line.strip()
self._tmp = ''.join(lines[i + 1:])
break
else: #or else it is a partial line
self._tmp = ''.join(lines[i:])
break
i += 1
else:
self._tmp = block
if self.raw_line:
self.parse_line()
# Decide if we have any headers.
line_ending = lines[0][len(self.raw_line):]
if self._tmp == line_ending:
#self._raw.append(self._tmp) -- why is this in here?
raise Complete
if self._tmp:
self.get_headers('')
def parse_line(self):
"""Parse and validate the Request-Line.
"""
# Tokenize the first line as a Request-Line.
# ==========================================
tokens = self.raw_line.strip().split()
if len(tokens) == 3:
method, uri, version = tokens
elif len(tokens) == 2:
method, uri = tokens
version = 'HTTP/0.9'
else:
raise Response(
400, "The Request-Line `%s' " % self.raw_line +
"appears to be malformed because it has " +
"neither two nor three tokens.")
# Validate the URI.
# =================
# We build a mapping using the urlparse naming convention for the keys.
# Then we do a little validation and cleanup.
uri = urlparse.urlparse(uri)
keys = ('scheme', 'netloc', 'path', 'parameters', 'query', 'fragment')
_uri = {}
for i in range(len(uri)):
k = keys[i]
v = uri[i]
_uri[k] = v
uri = _uri
if not uri['path']:
# this catches, e.g., '//foo'
raise Response(400)
if '%' in uri['path']:
uri['path'] = urllib.unquote(uri['path'])
# Validate the version.
# =====================
# Consider raising Response(505) here ... for 0.9? 2.0?
m = HTTP_VERSION.match(version)
if not m:
raise Response(
400, "The HTTP-Version `%s' appears to " % version +
"be malformed because it does not match the " +
"pattern `^HTTP/\d+\.\d+$'.")
# Save a few absolutely basic things for application use.
# =======================================================
self.method = method
self.uri = uri
self.path = uri['path']
# Headers
# =======
def get_headers(self, block):
"""The tricky part here is dealing with different line delimiters.
Technically MIME messages are \r\n delimited, but we can't assume that.
We want to split on the first double line break we get, regardless of
the control characters used.
"""
block = self._tmp + block
found_splitter = False
for double_line_break in ('\n\r\n', '\n\n'):
if double_line_break in block:
found_splitter = True
self.raw_headers, self._tmp = block.split(double_line_break, 1)
self.raw_headers = self.raw_headers.strip()
break
if not found_splitter:
self._tmp = block
self.headers = message_from_string(self.raw_headers)
if self.raw_headers:
self.get_body('')
# Body
# ====
def set_receiver(self):
"""Once the headers have been read, decide how to read the body.
We use the standard library's email package to parse the headers. The
receiver is stored on self so it persists across calls to our own
self.received, since it actually holds data between such calls. Later on
we will add the request body to self.message.
"""
content_length = int(self.headers.get('Content-Length', 0))
transfer_encoding = self.headers.get('Transfer-Encoding', '').lower()
if not content_length and not transfer_encoding:
self._receiver = None
elif content_length and not transfer_encoding:
buffer_ = OverflowableBuffer(self.adj.inbuf_overflow)
self._receiver = FixedStreamReceiver(content_length, buffer_)
elif transfer_encoding == 'chunked':
buffer_ = OverflowableBuffer(self.adj.inbuf_overflow)
self._receiver = ChunkedReceiver(buffer_)
else:
self._receiver = None
def get_body(self, block):
"""Given a block from an HTTP stream, build a message body.
We guarantee that the entire block is part of an HTTP request message.
"""
# Decide which receiver to use.
# =============================
# If we don't have a receiver, it means there is no body.
if self._receiver is None:
self.set_receiver()
if self._receiver is None:
self.raw_body = ''
raise Complete
# Get all of the message body in one block.
# =========================================
if self._tmp:
block = self._tmp + block
self._tmp = ''
if not self._receiver.completed:
self._receiver.received(block)
if not self._receiver.completed:
return # I'll be back!
# Store on self and get out of here.
# ==================================
# We toss the body away if the requested method doesn't allow for it. I
# read RFC 2616, Section 4.3 to call for this behavior rather than,
# e.g., returning a 400. I suppose the concern is that the input stream
# be properly positioned for multi-request connections.
#
# If anyone ever wants httpy to support extension-methods that take a
# body we will either have to add them here or make allowances in
# configuration.
if self.method in ('POST', 'POST'):
self.raw_body = self._receiver.getfile().read()
else:
self.raw_body = ''
raise Complete
class Bar:
implements(IBar)
foo = 1
class HTTPRequestParser(object):
"""A structure that collects the HTTP request.
Once the stream is completed, the instance is passed to
a server task constructor.
"""
implements(IStreamConsumer)
completed = 0 # Set once request is completed.
empty = 0 # Set if no request was made.
header_plus = ''
chunked = 0
content_length = 0
body_rcv = None
# Other attributes: first_line, header, headers, command, uri, version,
# path, query, fragment
# headers is a mapping containing keys translated to uppercase
# with dashes turned into underscores.
def __init__(self, adj):
"""
adj is an Adjustments object.
"""
self.headers = {}
self.adj = adj
def received(self, data):
"""
Receives the HTTP stream for one request.
Returns the number of bytes consumed.
Sets the completed flag once both the header and the
body have been received.
"""
if self.completed:
return 0 # Can't consume any more.
datalen = len(data)
br = self.body_rcv
if br is None:
# In header.
s = self.header_plus + data
index = find_double_newline(s)
if index >= 0:
# Header finished.
header_plus = s[:index]
consumed = len(data) - (len(s) - index)
self.in_header = 0
# Remove preceeding blank lines.
header_plus = header_plus.lstrip()
if not header_plus:
self.empty = 1
self.completed = 1
else:
self.parse_header(header_plus)
if self.body_rcv is None:
self.completed = 1
return consumed
else:
# Header not finished yet.
self.header_plus = s
return datalen
else:
# In body.
consumed = br.received(data)
if br.completed:
self.completed = 1
return consumed
def parse_header(self, header_plus):
"""
Parses the header_plus block of text (the headers plus the
first line of the request).
"""
index = header_plus.find('\n')
if index >= 0:
first_line = header_plus[:index].rstrip()
header = header_plus[index + 1:]
else:
first_line = header_plus.rstrip()
header = ''
self.first_line = first_line
self.header = header
lines = self.get_header_lines()
headers = self.headers
for line in lines:
index = line.find(':')
if index > 0:
key = line[:index]
value = line[index + 1:].strip()
key1 = key.upper().replace('-', '_')
# If a header already exists, we append subsequent values
# seperated by a comma. Applications already need to handle
# the comma seperated values, as HTTP front ends might do
# the concatenation for you (behavior specified in RFC2616).
try:
headers[key1] += ', %s' % value
except KeyError:
headers[key1] = value
# else there's garbage in the headers?
command, uri, version = self.crack_first_line()
self.command = str(command)
self.uri = str(uri)
self.version = version
self.split_uri()
if version == '1.1':
te = headers.get('TRANSFER_ENCODING', '')
if te == 'chunked':
from httpy._zope.server.http.chunking import ChunkedReceiver
self.chunked = 1
buf = OverflowableBuffer(self.adj.inbuf_overflow)
self.body_rcv = ChunkedReceiver(buf)
if not self.chunked:
try:
cl = int(headers.get('CONTENT_LENGTH', 0))
except ValueError:
cl = 0
self.content_length = cl
if cl > 0:
buf = OverflowableBuffer(self.adj.inbuf_overflow)
self.body_rcv = FixedStreamReceiver(cl, buf)
def get_header_lines(self):
"""
Splits the header into lines, putting multi-line headers together.
"""
r = []
lines = self.header.split('\n')
for line in lines:
if line and line[0] in ' \t':
r[-1] = r[-1] + line[1:]
else:
r.append(line)
return r
first_line_re = re.compile(
'([^ ]+) (?:[^ :?#]+://[^ ?#/]*)?([^ ]+)(( HTTP/([0-9.]+))$|$)')
def crack_first_line(self):
r = self.first_line
m = self.first_line_re.match(r)
if m is not None and m.end() == len(r):
if m.group(3):
version = m.group(5)
else:
version = None
return m.group(1).upper(), m.group(2), version
else:
return None, None, None
path_regex = re.compile(
# path query fragment
r'([^?#]*)(\?[^#]*)?(#.*)?')
def split_uri(self):
m = self.path_regex.match(self.uri)
if m.end() != len(self.uri):
raise ValueError, "Broken URI"
else:
path, query, self.fragment = m.groups()
if path and '%' in path:
path = unquote(path)
self.path = path
if query:
query = query[1:]
self.query = query
def getBodyStream(self):
body_rcv = self.body_rcv
if body_rcv is not None:
return body_rcv.getfile()
else:
return StringIO('')
class C(object):
implements(I)
class C(object):
def f(self, b): pass
implements(I)
