class File(StaticRenderMixin):
"""
File is a resource that represents a plain non-interpreted file
(although it can look for an extension like .rpy or .cgi and hand the
file to a processor for interpretation if you wish). Its constructor
takes a file path.
Alternatively, you can give a directory path to the constructor. In this
case the resource will represent that directory, and its children will
be files underneath that directory. This provides access to an entire
filesystem tree with a single Resource.
If you map the URL C{http://server/FILE} to a resource created as
File('/tmp'), C{http://server/FILE/foo/bar.html} will return the contents of
C{/tmp/foo/bar.html} .
"""
implements(iweb.IResource)
def _getContentTypes(self):
if not hasattr(File, "_sharedContentTypes"):
File._sharedContentTypes = loadMimeTypes()
return File._sharedContentTypes
contentTypes = property(_getContentTypes)
contentEncodings = {".gz": "gzip", ".bz2": "bzip2"}
processors = {}
indexNames = ["index", "index.html", "index.htm", "index.trp", "index.rpy"]
type = None
def __init__(self,
path,
defaultType="text/plain",
ignoredExts=(),
processors=None,
indexNames=None):
"""Create a file with the given path.
"""
super(File, self).__init__()
self.putChildren = {}
if isinstance(path, FilePath):
self.fp = path
else:
assert isinstance(path, str), "This should be a string."
self.fp = FilePath(path)
# Remove the dots from the path to split
self.defaultType = defaultType
self.ignoredExts = list(ignoredExts)
if processors is not None:
self.processors = dict([(key.lower(), value)
for key, value in processors.items()])
if indexNames is not None:
self.indexNames = indexNames
def comparePath(self, path):
if isinstance(path, FilePath):
return path.path == self.fp.path
else:
return path == self.fp.path
def exists(self):
return self.fp.exists()
def etag(self):
if not self.fp.exists(): return succeed(None)
st = self.fp.statinfo
#
# Mark ETag as weak if it was modified more recently than we can
# measure and report, as it could be modified again in that span
# and we then wouldn't know to provide a new ETag.
#
weak = (time.time() - st.st_mtime <= 1)
return succeed(
http_headers.ETag("%X-%X-%X" %
(st.st_ino, st.st_size, st.st_mtime),
weak=weak))
def lastModified(self):
if self.fp.exists():
return self.fp.getmtime()
else:
return None
def creationDate(self):
if self.fp.exists():
return self.fp.getmtime()
else:
return None
def contentLength(self):
if self.fp.exists():
if self.fp.isfile():
return self.fp.getsize()
else:
# Computing this would require rendering the resource; let's
# punt instead.
return None
else:
return None
def _initTypeAndEncoding(self):
self._type, self._encoding = getTypeAndEncoding(
self.fp.basename(), self.contentTypes, self.contentEncodings,
self.defaultType)
# Handle cases not covered by getTypeAndEncoding()
if self.fp.isdir(): self._type = "httpd/unix-directory"
def contentType(self):
if not hasattr(self, "_type"):
self._initTypeAndEncoding()
return http_headers.MimeType.fromString(self._type)
def contentEncoding(self):
if not hasattr(self, "_encoding"):
self._initTypeAndEncoding()
return self._encoding
def displayName(self):
if self.fp.exists():
return self.fp.basename()
else:
return None
def ignoreExt(self, ext):
"""Ignore the given extension.
Serve file.ext if file is requested
"""
self.ignoredExts.append(ext)
def putChild(self, name, child):
"""
Register a child with the given name with this resource.
@param name: the name of the child (a URI path segment)
@param child: the child to register
"""
self.putChildren[name] = child
def getChild(self, name):
"""
Look up a child resource.
@return: the child of this resource with the given name.
"""
if name == "":
return self
child = self.putChildren.get(name, None)
if child: return child
child_fp = self.fp.child(name)
if hasattr(self, "knownChildren"):
if name in self.knownChildren:
child_fp.existsCached = True
if child_fp.exists():
return self.createSimilarFile(child_fp)
else:
return None
def listChildren(self):
"""
@return: a sequence of the names of all known children of this resource.
"""
children = self.putChildren.keys()
if self.fp.isdir():
children += [c for c in self.fp.listdir() if c not in children]
self.knownChildren = set(children)
return children
def locateChild(self, req, segments):
"""
See L{IResource}C{.locateChild}.
"""
# If getChild() finds a child resource, return it
child = self.getChild(segments[0])
if child is not None: return (child, segments[1:])
# If we're not backed by a directory, we have no children.
# But check for existance first; we might be a collection resource
# that the request wants created.
self.fp.restat(False)
if self.fp.exists() and not self.fp.isdir(): return (None, ())
# OK, we need to return a child corresponding to the first segment
path = segments[0]
if path:
fpath = self.fp.child(path)
else:
# Request is for a directory (collection) resource
return (self, server.StopTraversal)
# Don't run processors on directories - if someone wants their own
# customized directory rendering, subclass File instead.
if fpath.isfile():
processor = self.processors.get(fpath.splitext()[1].lower())
if processor:
return (processor(fpath.path), segments[1:])
elif not fpath.exists():
sibling_fpath = fpath.siblingExtensionSearch(*self.ignoredExts)
if sibling_fpath is not None:
fpath = sibling_fpath
return self.createSimilarFile(fpath.path), segments[1:]
def renderHTTP(self, req):
self.fp.changed()
return super(File, self).renderHTTP(req)
def render(self, req):
"""You know what you doing."""
if not self.fp.exists():
return responsecode.NOT_FOUND
if self.fp.isdir():
if req.path[-1] != "/":
# Redirect to include trailing '/' in URI
return http.RedirectResponse(
req.unparseURL(path=req.path + '/'))
else:
ifp = self.fp.childSearchPreauth(*self.indexNames)
if ifp:
# Render from the index file
standin = self.createSimilarFile(ifp.path)
else:
# Directory listing is in twistedcaldav.extensions
standin = Data(
"\n".join(["Directory: " + str(req.path), "---"] + [
x.basename() + ("/" if x.isdir() else "")
for x in self.fp.children()
]), "text/plain")
return standin.render(req)
try:
f = self.fp.open()
except IOError, e:
import errno
if e[0] == errno.EACCES:
return responsecode.FORBIDDEN
elif e[0] == errno.ENOENT:
return responsecode.NOT_FOUND
else:
raise
response = http.Response()
response.stream = stream.FileStream(f, 0, self.fp.getsize())
for (header, value) in (
("content-type", self.contentType()),
("content-encoding", self.contentEncoding()),
):
if value is not None:
response.headers.setHeader(header, value)
return response
class File(StaticRenderMixin):
"""
File is a resource that represents a plain non-interpreted file
(although it can look for an extension like .rpy or .cgi and hand the
file to a processor for interpretation if you wish). Its constructor
takes a file path.
Alternatively, you can give a directory path to the constructor. In this
case the resource will represent that directory, and its children will
be files underneath that directory. This provides access to an entire
filesystem tree with a single Resource.
If you map the URL C{http://server/FILE} to a resource created as
File('/tmp'), C{http://server/FILE/foo/bar.html} will return the contents of
C{/tmp/foo/bar.html} .
"""
implements(iweb.IResource)
def _getContentTypes(self):
if not hasattr(File, "_sharedContentTypes"):
File._sharedContentTypes = loadMimeTypes()
return File._sharedContentTypes
contentTypes = property(_getContentTypes)
contentEncodings = {
".gz" : "gzip",
".bz2": "bzip2"
}
processors = {}
indexNames = ["index", "index.html", "index.htm", "index.trp", "index.rpy"]
type = None
def __init__(self, path, defaultType="text/plain", ignoredExts=(), processors=None, indexNames=None):
"""Create a file with the given path.
"""
super(File, self).__init__()
self.putChildren = {}
if isinstance(path, FilePath):
self.fp = path
else:
assert isinstance(path, str), "This should be a string."
self.fp = FilePath(path)
# Remove the dots from the path to split
self.defaultType = defaultType
self.ignoredExts = list(ignoredExts)
if processors is not None:
self.processors = dict([
(key.lower(), value)
for key, value in processors.items()
])
if indexNames is not None:
self.indexNames = indexNames
def comparePath(self, path):
if isinstance(path, FilePath):
return path.path == self.fp.path
else:
return path == self.fp.path
def exists(self):
return self.fp.exists()
def etag(self):
if not self.fp.exists(): return succeed(None)
st = self.fp.statinfo
#
# Mark ETag as weak if it was modified more recently than we can
# measure and report, as it could be modified again in that span
# and we then wouldn't know to provide a new ETag.
#
weak = (time.time() - st.st_mtime <= 1)
return succeed(http_headers.ETag(
"%X-%X-%X" % (st.st_ino, st.st_size, st.st_mtime),
weak=weak
))
def lastModified(self):
if self.fp.exists():
return self.fp.getmtime()
else:
return None
def creationDate(self):
if self.fp.exists():
return self.fp.getmtime()
else:
return None
def contentLength(self):
if self.fp.exists():
if self.fp.isfile():
return self.fp.getsize()
else:
# Computing this would require rendering the resource; let's
# punt instead.
return None
else:
return None
def _initTypeAndEncoding(self):
self._type, self._encoding = getTypeAndEncoding(
self.fp.basename(),
self.contentTypes,
self.contentEncodings,
self.defaultType
)
# Handle cases not covered by getTypeAndEncoding()
if self.fp.isdir(): self._type = "httpd/unix-directory"
def contentType(self):
if not hasattr(self, "_type"):
self._initTypeAndEncoding()
return http_headers.MimeType.fromString(self._type)
def contentEncoding(self):
if not hasattr(self, "_encoding"):
self._initTypeAndEncoding()
return self._encoding
def displayName(self):
if self.fp.exists():
return self.fp.basename()
else:
return None
def ignoreExt(self, ext):
"""Ignore the given extension.
Serve file.ext if file is requested
"""
self.ignoredExts.append(ext)
def putChild(self, name, child):
"""
Register a child with the given name with this resource.
@param name: the name of the child (a URI path segment)
@param child: the child to register
"""
self.putChildren[name] = child
def getChild(self, name):
"""
Look up a child resource.
@return: the child of this resource with the given name.
"""
if name == "":
return self
child = self.putChildren.get(name, None)
if child: return child
child_fp = self.fp.child(name)
if hasattr(self, "knownChildren"):
if name in self.knownChildren:
child_fp.existsCached = True
if child_fp.exists():
return self.createSimilarFile(child_fp)
else:
return None
def listChildren(self):
"""
@return: a sequence of the names of all known children of this resource.
"""
children = self.putChildren.keys()
if self.fp.isdir():
children += [c for c in self.fp.listdir() if c not in children]
self.knownChildren = set(children)
return children
def locateChild(self, req, segments):
"""
See L{IResource}C{.locateChild}.
"""
# If getChild() finds a child resource, return it
child = self.getChild(segments[0])
if child is not None: return (child, segments[1:])
# If we're not backed by a directory, we have no children.
# But check for existance first; we might be a collection resource
# that the request wants created.
self.fp.restat(False)
if self.fp.exists() and not self.fp.isdir(): return (None, ())
# OK, we need to return a child corresponding to the first segment
path = segments[0]
if path:
fpath = self.fp.child(path)
else:
# Request is for a directory (collection) resource
return (self, server.StopTraversal)
# Don't run processors on directories - if someone wants their own
# customized directory rendering, subclass File instead.
if fpath.isfile():
processor = self.processors.get(fpath.splitext()[1].lower())
if processor:
return (
processor(fpath.path),
segments[1:])
elif not fpath.exists():
sibling_fpath = fpath.siblingExtensionSearch(*self.ignoredExts)
if sibling_fpath is not None:
fpath = sibling_fpath
return self.createSimilarFile(fpath.path), segments[1:]
def renderHTTP(self, req):
self.fp.changed()
return super(File, self).renderHTTP(req)
def render(self, req):
"""You know what you doing."""
if not self.fp.exists():
return responsecode.NOT_FOUND
if self.fp.isdir():
if req.path[-1] != "/":
# Redirect to include trailing '/' in URI
return http.RedirectResponse(req.unparseURL(path=req.path+'/'))
else:
ifp = self.fp.childSearchPreauth(*self.indexNames)
if ifp:
# Render from the index file
standin = self.createSimilarFile(ifp.path)
else:
# Directory listing is in twistedcaldav.extensions
standin = Data(
"\n".join(["Directory: " + str(req.path), "---"] +
[x.basename() + ("/" if x.isdir() else "")
for x in self.fp.children()]),
"text/plain")
return standin.render(req)
try:
f = self.fp.open()
except IOError, e:
import errno
if e[0] == errno.EACCES:
return responsecode.FORBIDDEN
elif e[0] == errno.ENOENT:
return responsecode.NOT_FOUND
else:
raise
response = http.Response()
response.stream = stream.FileStream(f, 0, self.fp.getsize())
for (header, value) in (
("content-type", self.contentType()),
("content-encoding", self.contentEncoding()),
):
if value is not None:
response.headers.setHeader(header, value)
return response
class EINVALTestCase(TestCase):
"""
Sometimes, L{os.listdir} will raise C{EINVAL}. This is a transient error,
and L{CachingFilePath.listdir} should work around it by retrying the
C{listdir} operation until it succeeds.
"""
def setUp(self):
"""
Create a L{CachingFilePath} for the test to use.
"""
self.cfp = CachingFilePath(self.mktemp())
self.clock = Clock()
self.cfp._sleep = self.clock.advance
def test_testValidity(self):
"""
If C{listdir} is replaced on a L{CachingFilePath}, we should be able to
observe exceptions raised by the replacement. This verifies that the
test patching done here is actually testing something.
"""
class CustomException(Exception):
"Just for testing."
def blowUp(dirname):
raise CustomException()
self.cfp._listdir = blowUp
self.assertRaises(CustomException, self.cfp.listdir)
self.assertRaises(CustomException, self.cfp.children)
def test_retryLoop(self):
"""
L{CachingFilePath} should catch C{EINVAL} and respond by retrying the
C{listdir} operation until it succeeds.
"""
calls = []
def raiseEINVAL(dirname):
calls.append(dirname)
if len(calls) < 5:
raise OSError(EINVAL, "This should be caught by the test.")
return ['a', 'b', 'c']
self.cfp._listdir = raiseEINVAL
self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
self.assertEquals(self.cfp.children(), [
CachingFilePath(pathjoin(self.cfp.path, 'a')),
CachingFilePath(pathjoin(self.cfp.path, 'b')),
CachingFilePath(pathjoin(self.cfp.path, 'c')),
])
def requireTimePassed(self, filenames):
"""
Create a replacement for listdir() which only fires after a certain
amount of time.
"""
self.calls = []
def thunk(dirname):
now = self.clock.seconds()
if now < 20.0:
self.calls.append(now)
raise OSError(EINVAL, "Not enough time has passed yet.")
else:
return filenames
self.cfp._listdir = thunk
def assertRequiredTimePassed(self):
"""
Assert that calls to the simulated time.sleep() installed by
C{requireTimePassed} have been invoked the required number of times.
"""
# Waiting should be growing by *2 each time until the additional wait
# exceeds BACKOFF_MAX (5), at which point we should wait for 5s each
# time.
def cumulative(values):
current = 0.0
for value in values:
current += value
yield current
self.assertEquals(
self.calls,
list(cumulative(
[0.0, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0]
))
)
def test_backoff(self):
"""
L{CachingFilePath} will wait for an increasing interval up to
C{BACKOFF_MAX} between calls to listdir().
"""
self.requireTimePassed(['a', 'b', 'c'])
self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
class EINVALTestCase(TestCase):
"""
Sometimes, L{os.listdir} will raise C{EINVAL}. This is a transient error,
and L{CachingFilePath.listdir} should work around it by retrying the
C{listdir} operation until it succeeds.
"""
def setUp(self):
"""
Create a L{CachingFilePath} for the test to use.
"""
self.cfp = CachingFilePath(self.mktemp())
self.clock = Clock()
self.cfp._sleep = self.clock.advance
def test_testValidity(self):
"""
If C{listdir} is replaced on a L{CachingFilePath}, we should be able to
observe exceptions raised by the replacement. This verifies that the
test patching done here is actually testing something.
"""
class CustomException(Exception): "Just for testing."
def blowUp(dirname):
raise CustomException()
self.cfp._listdir = blowUp
self.assertRaises(CustomException, self.cfp.listdir)
self.assertRaises(CustomException, self.cfp.children)
def test_retryLoop(self):
"""
L{CachingFilePath} should catch C{EINVAL} and respond by retrying the
C{listdir} operation until it succeeds.
"""
calls = []
def raiseEINVAL(dirname):
calls.append(dirname)
if len(calls) < 5:
raise OSError(EINVAL, "This should be caught by the test.")
return ['a', 'b', 'c']
self.cfp._listdir = raiseEINVAL
self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
self.assertEquals(self.cfp.children(), [
CachingFilePath(pathjoin(self.cfp.path, 'a')),
CachingFilePath(pathjoin(self.cfp.path, 'b')),
CachingFilePath(pathjoin(self.cfp.path, 'c')),])
def requireTimePassed(self, filenames):
"""
Create a replacement for listdir() which only fires after a certain
amount of time.
"""
self.calls = []
def thunk(dirname):
now = self.clock.seconds()
if now < 20.0:
self.calls.append(now)
raise OSError(EINVAL, "Not enough time has passed yet.")
else:
return filenames
self.cfp._listdir = thunk
def assertRequiredTimePassed(self):
"""
Assert that calls to the simulated time.sleep() installed by
C{requireTimePassed} have been invoked the required number of times.
"""
# Waiting should be growing by *2 each time until the additional wait
# exceeds BACKOFF_MAX (5), at which point we should wait for 5s each
# time.
def cumulative(values):
current = 0.0
for value in values:
current += value
yield current
self.assertEquals(self.calls,
list(cumulative(
[0.0, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0])))
def test_backoff(self):
"""
L{CachingFilePath} will wait for an increasing interval up to
C{BACKOFF_MAX} between calls to listdir().
"""
self.requireTimePassed(['a', 'b', 'c'])
self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c'])
def test_siblingExtensionSearch(self):
"""
L{FilePath.siblingExtensionSearch} is unfortunately not implemented in
terms of L{FilePath.listdir}, so we need to verify that it will also
retry.
"""
filenames = [self.cfp.basename()+'.a',
self.cfp.basename() + '.b',
self.cfp.basename() + '.c']
siblings = map(self.cfp.sibling, filenames)
for sibling in siblings:
sibling.touch()
self.requireTimePassed(filenames)
self.assertEquals(self.cfp.siblingExtensionSearch("*"),
siblings[0])
self.assertRequiredTimePassed()
