class File(StaticRenderMixin): """ File is a resource that represents a plain non-interpreted file (although it can look for an extension like .rpy or .cgi and hand the file to a processor for interpretation if you wish). Its constructor takes a file path. Alternatively, you can give a directory path to the constructor. In this case the resource will represent that directory, and its children will be files underneath that directory. This provides access to an entire filesystem tree with a single Resource. If you map the URL C{http://server/FILE} to a resource created as File('/tmp'), C{http://server/FILE/foo/bar.html} will return the contents of C{/tmp/foo/bar.html} . """ implements(iweb.IResource) def _getContentTypes(self): if not hasattr(File, "_sharedContentTypes"): File._sharedContentTypes = loadMimeTypes() return File._sharedContentTypes contentTypes = property(_getContentTypes) contentEncodings = {".gz": "gzip", ".bz2": "bzip2"} processors = {} indexNames = ["index", "index.html", "index.htm", "index.trp", "index.rpy"] type = None def __init__(self, path, defaultType="text/plain", ignoredExts=(), processors=None, indexNames=None): """Create a file with the given path. """ super(File, self).__init__() self.putChildren = {} if isinstance(path, FilePath): self.fp = path else: assert isinstance(path, str), "This should be a string." self.fp = FilePath(path) # Remove the dots from the path to split self.defaultType = defaultType self.ignoredExts = list(ignoredExts) if processors is not None: self.processors = dict([(key.lower(), value) for key, value in processors.items()]) if indexNames is not None: self.indexNames = indexNames def comparePath(self, path): if isinstance(path, FilePath): return path.path == self.fp.path else: return path == self.fp.path def exists(self): return self.fp.exists() def etag(self): if not self.fp.exists(): return succeed(None) st = self.fp.statinfo # # Mark ETag as weak if it was modified more recently than we can # measure and report, as it could be modified again in that span # and we then wouldn't know to provide a new ETag. # weak = (time.time() - st.st_mtime <= 1) return succeed( http_headers.ETag("%X-%X-%X" % (st.st_ino, st.st_size, st.st_mtime), weak=weak)) def lastModified(self): if self.fp.exists(): return self.fp.getmtime() else: return None def creationDate(self): if self.fp.exists(): return self.fp.getmtime() else: return None def contentLength(self): if self.fp.exists(): if self.fp.isfile(): return self.fp.getsize() else: # Computing this would require rendering the resource; let's # punt instead. return None else: return None def _initTypeAndEncoding(self): self._type, self._encoding = getTypeAndEncoding( self.fp.basename(), self.contentTypes, self.contentEncodings, self.defaultType) # Handle cases not covered by getTypeAndEncoding() if self.fp.isdir(): self._type = "httpd/unix-directory" def contentType(self): if not hasattr(self, "_type"): self._initTypeAndEncoding() return http_headers.MimeType.fromString(self._type) def contentEncoding(self): if not hasattr(self, "_encoding"): self._initTypeAndEncoding() return self._encoding def displayName(self): if self.fp.exists(): return self.fp.basename() else: return None def ignoreExt(self, ext): """Ignore the given extension. Serve file.ext if file is requested """ self.ignoredExts.append(ext) def putChild(self, name, child): """ Register a child with the given name with this resource. @param name: the name of the child (a URI path segment) @param child: the child to register """ self.putChildren[name] = child def getChild(self, name): """ Look up a child resource. @return: the child of this resource with the given name. """ if name == "": return self child = self.putChildren.get(name, None) if child: return child child_fp = self.fp.child(name) if hasattr(self, "knownChildren"): if name in self.knownChildren: child_fp.existsCached = True if child_fp.exists(): return self.createSimilarFile(child_fp) else: return None def listChildren(self): """ @return: a sequence of the names of all known children of this resource. """ children = self.putChildren.keys() if self.fp.isdir(): children += [c for c in self.fp.listdir() if c not in children] self.knownChildren = set(children) return children def locateChild(self, req, segments): """ See L{IResource}C{.locateChild}. """ # If getChild() finds a child resource, return it child = self.getChild(segments[0]) if child is not None: return (child, segments[1:]) # If we're not backed by a directory, we have no children. # But check for existance first; we might be a collection resource # that the request wants created. self.fp.restat(False) if self.fp.exists() and not self.fp.isdir(): return (None, ()) # OK, we need to return a child corresponding to the first segment path = segments[0] if path: fpath = self.fp.child(path) else: # Request is for a directory (collection) resource return (self, server.StopTraversal) # Don't run processors on directories - if someone wants their own # customized directory rendering, subclass File instead. if fpath.isfile(): processor = self.processors.get(fpath.splitext()[1].lower()) if processor: return (processor(fpath.path), segments[1:]) elif not fpath.exists(): sibling_fpath = fpath.siblingExtensionSearch(*self.ignoredExts) if sibling_fpath is not None: fpath = sibling_fpath return self.createSimilarFile(fpath.path), segments[1:] def renderHTTP(self, req): self.fp.changed() return super(File, self).renderHTTP(req) def render(self, req): """You know what you doing.""" if not self.fp.exists(): return responsecode.NOT_FOUND if self.fp.isdir(): if req.path[-1] != "/": # Redirect to include trailing '/' in URI return http.RedirectResponse( req.unparseURL(path=req.path + '/')) else: ifp = self.fp.childSearchPreauth(*self.indexNames) if ifp: # Render from the index file standin = self.createSimilarFile(ifp.path) else: # Directory listing is in twistedcaldav.extensions standin = Data( "\n".join(["Directory: " + str(req.path), "---"] + [ x.basename() + ("/" if x.isdir() else "") for x in self.fp.children() ]), "text/plain") return standin.render(req) try: f = self.fp.open() except IOError, e: import errno if e[0] == errno.EACCES: return responsecode.FORBIDDEN elif e[0] == errno.ENOENT: return responsecode.NOT_FOUND else: raise response = http.Response() response.stream = stream.FileStream(f, 0, self.fp.getsize()) for (header, value) in ( ("content-type", self.contentType()), ("content-encoding", self.contentEncoding()), ): if value is not None: response.headers.setHeader(header, value) return response
class EINVALTestCase(TestCase): """ Sometimes, L{os.listdir} will raise C{EINVAL}. This is a transient error, and L{CachingFilePath.listdir} should work around it by retrying the C{listdir} operation until it succeeds. """ def setUp(self): """ Create a L{CachingFilePath} for the test to use. """ self.cfp = CachingFilePath(self.mktemp()) self.clock = Clock() self.cfp._sleep = self.clock.advance def test_testValidity(self): """ If C{listdir} is replaced on a L{CachingFilePath}, we should be able to observe exceptions raised by the replacement. This verifies that the test patching done here is actually testing something. """ class CustomException(Exception): "Just for testing." def blowUp(dirname): raise CustomException() self.cfp._listdir = blowUp self.assertRaises(CustomException, self.cfp.listdir) self.assertRaises(CustomException, self.cfp.children) def test_retryLoop(self): """ L{CachingFilePath} should catch C{EINVAL} and respond by retrying the C{listdir} operation until it succeeds. """ calls = [] def raiseEINVAL(dirname): calls.append(dirname) if len(calls) < 5: raise OSError(EINVAL, "This should be caught by the test.") return ['a', 'b', 'c'] self.cfp._listdir = raiseEINVAL self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c']) self.assertEquals(self.cfp.children(), [ CachingFilePath(pathjoin(self.cfp.path, 'a')), CachingFilePath(pathjoin(self.cfp.path, 'b')), CachingFilePath(pathjoin(self.cfp.path, 'c')),]) def requireTimePassed(self, filenames): """ Create a replacement for listdir() which only fires after a certain amount of time. """ self.calls = [] def thunk(dirname): now = self.clock.seconds() if now < 20.0: self.calls.append(now) raise OSError(EINVAL, "Not enough time has passed yet.") else: return filenames self.cfp._listdir = thunk def assertRequiredTimePassed(self): """ Assert that calls to the simulated time.sleep() installed by C{requireTimePassed} have been invoked the required number of times. """ # Waiting should be growing by *2 each time until the additional wait # exceeds BACKOFF_MAX (5), at which point we should wait for 5s each # time. def cumulative(values): current = 0.0 for value in values: current += value yield current self.assertEquals(self.calls, list(cumulative( [0.0, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0]))) def test_backoff(self): """ L{CachingFilePath} will wait for an increasing interval up to C{BACKOFF_MAX} between calls to listdir(). """ self.requireTimePassed(['a', 'b', 'c']) self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c']) def test_siblingExtensionSearch(self): """ L{FilePath.siblingExtensionSearch} is unfortunately not implemented in terms of L{FilePath.listdir}, so we need to verify that it will also retry. """ filenames = [self.cfp.basename()+'.a', self.cfp.basename() + '.b', self.cfp.basename() + '.c'] siblings = map(self.cfp.sibling, filenames) for sibling in siblings: sibling.touch() self.requireTimePassed(filenames) self.assertEquals(self.cfp.siblingExtensionSearch("*"), siblings[0]) self.assertRequiredTimePassed()
class File(StaticRenderMixin): """ File is a resource that represents a plain non-interpreted file (although it can look for an extension like .rpy or .cgi and hand the file to a processor for interpretation if you wish). Its constructor takes a file path. Alternatively, you can give a directory path to the constructor. In this case the resource will represent that directory, and its children will be files underneath that directory. This provides access to an entire filesystem tree with a single Resource. If you map the URL C{http://server/FILE} to a resource created as File('/tmp'), C{http://server/FILE/foo/bar.html} will return the contents of C{/tmp/foo/bar.html} . """ implements(iweb.IResource) def _getContentTypes(self): if not hasattr(File, "_sharedContentTypes"): File._sharedContentTypes = loadMimeTypes() return File._sharedContentTypes contentTypes = property(_getContentTypes) contentEncodings = { ".gz" : "gzip", ".bz2": "bzip2" } processors = {} indexNames = ["index", "index.html", "index.htm", "index.trp", "index.rpy"] type = None def __init__(self, path, defaultType="text/plain", ignoredExts=(), processors=None, indexNames=None): """Create a file with the given path. """ super(File, self).__init__() self.putChildren = {} if isinstance(path, FilePath): self.fp = path else: assert isinstance(path, str), "This should be a string." self.fp = FilePath(path) # Remove the dots from the path to split self.defaultType = defaultType self.ignoredExts = list(ignoredExts) if processors is not None: self.processors = dict([ (key.lower(), value) for key, value in processors.items() ]) if indexNames is not None: self.indexNames = indexNames def comparePath(self, path): if isinstance(path, FilePath): return path.path == self.fp.path else: return path == self.fp.path def exists(self): return self.fp.exists() def etag(self): if not self.fp.exists(): return succeed(None) st = self.fp.statinfo # # Mark ETag as weak if it was modified more recently than we can # measure and report, as it could be modified again in that span # and we then wouldn't know to provide a new ETag. # weak = (time.time() - st.st_mtime <= 1) return succeed(http_headers.ETag( "%X-%X-%X" % (st.st_ino, st.st_size, st.st_mtime), weak=weak )) def lastModified(self): if self.fp.exists(): return self.fp.getmtime() else: return None def creationDate(self): if self.fp.exists(): return self.fp.getmtime() else: return None def contentLength(self): if self.fp.exists(): if self.fp.isfile(): return self.fp.getsize() else: # Computing this would require rendering the resource; let's # punt instead. return None else: return None def _initTypeAndEncoding(self): self._type, self._encoding = getTypeAndEncoding( self.fp.basename(), self.contentTypes, self.contentEncodings, self.defaultType ) # Handle cases not covered by getTypeAndEncoding() if self.fp.isdir(): self._type = "httpd/unix-directory" def contentType(self): if not hasattr(self, "_type"): self._initTypeAndEncoding() return http_headers.MimeType.fromString(self._type) def contentEncoding(self): if not hasattr(self, "_encoding"): self._initTypeAndEncoding() return self._encoding def displayName(self): if self.fp.exists(): return self.fp.basename() else: return None def ignoreExt(self, ext): """Ignore the given extension. Serve file.ext if file is requested """ self.ignoredExts.append(ext) def putChild(self, name, child): """ Register a child with the given name with this resource. @param name: the name of the child (a URI path segment) @param child: the child to register """ self.putChildren[name] = child def getChild(self, name): """ Look up a child resource. @return: the child of this resource with the given name. """ if name == "": return self child = self.putChildren.get(name, None) if child: return child child_fp = self.fp.child(name) if hasattr(self, "knownChildren"): if name in self.knownChildren: child_fp.existsCached = True if child_fp.exists(): return self.createSimilarFile(child_fp) else: return None def listChildren(self): """ @return: a sequence of the names of all known children of this resource. """ children = self.putChildren.keys() if self.fp.isdir(): children += [c for c in self.fp.listdir() if c not in children] self.knownChildren = set(children) return children def locateChild(self, req, segments): """ See L{IResource}C{.locateChild}. """ # If getChild() finds a child resource, return it child = self.getChild(segments[0]) if child is not None: return (child, segments[1:]) # If we're not backed by a directory, we have no children. # But check for existance first; we might be a collection resource # that the request wants created. self.fp.restat(False) if self.fp.exists() and not self.fp.isdir(): return (None, ()) # OK, we need to return a child corresponding to the first segment path = segments[0] if path: fpath = self.fp.child(path) else: # Request is for a directory (collection) resource return (self, server.StopTraversal) # Don't run processors on directories - if someone wants their own # customized directory rendering, subclass File instead. if fpath.isfile(): processor = self.processors.get(fpath.splitext()[1].lower()) if processor: return ( processor(fpath.path), segments[1:]) elif not fpath.exists(): sibling_fpath = fpath.siblingExtensionSearch(*self.ignoredExts) if sibling_fpath is not None: fpath = sibling_fpath return self.createSimilarFile(fpath.path), segments[1:] def renderHTTP(self, req): self.fp.changed() return super(File, self).renderHTTP(req) def render(self, req): """You know what you doing.""" if not self.fp.exists(): return responsecode.NOT_FOUND if self.fp.isdir(): if req.path[-1] != "/": # Redirect to include trailing '/' in URI return http.RedirectResponse(req.unparseURL(path=req.path+'/')) else: ifp = self.fp.childSearchPreauth(*self.indexNames) if ifp: # Render from the index file standin = self.createSimilarFile(ifp.path) else: # Directory listing is in twistedcaldav.extensions standin = Data( "\n".join(["Directory: " + str(req.path), "---"] + [x.basename() + ("/" if x.isdir() else "") for x in self.fp.children()]), "text/plain") return standin.render(req) try: f = self.fp.open() except IOError, e: import errno if e[0] == errno.EACCES: return responsecode.FORBIDDEN elif e[0] == errno.ENOENT: return responsecode.NOT_FOUND else: raise response = http.Response() response.stream = stream.FileStream(f, 0, self.fp.getsize()) for (header, value) in ( ("content-type", self.contentType()), ("content-encoding", self.contentEncoding()), ): if value is not None: response.headers.setHeader(header, value) return response
class EINVALTestCase(TestCase): """ Sometimes, L{os.listdir} will raise C{EINVAL}. This is a transient error, and L{CachingFilePath.listdir} should work around it by retrying the C{listdir} operation until it succeeds. """ def setUp(self): """ Create a L{CachingFilePath} for the test to use. """ self.cfp = CachingFilePath(self.mktemp()) self.clock = Clock() self.cfp._sleep = self.clock.advance def test_testValidity(self): """ If C{listdir} is replaced on a L{CachingFilePath}, we should be able to observe exceptions raised by the replacement. This verifies that the test patching done here is actually testing something. """ class CustomException(Exception): "Just for testing." def blowUp(dirname): raise CustomException() self.cfp._listdir = blowUp self.assertRaises(CustomException, self.cfp.listdir) self.assertRaises(CustomException, self.cfp.children) def test_retryLoop(self): """ L{CachingFilePath} should catch C{EINVAL} and respond by retrying the C{listdir} operation until it succeeds. """ calls = [] def raiseEINVAL(dirname): calls.append(dirname) if len(calls) < 5: raise OSError(EINVAL, "This should be caught by the test.") return ['a', 'b', 'c'] self.cfp._listdir = raiseEINVAL self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c']) self.assertEquals(self.cfp.children(), [ CachingFilePath(pathjoin(self.cfp.path, 'a')), CachingFilePath(pathjoin(self.cfp.path, 'b')), CachingFilePath(pathjoin(self.cfp.path, 'c')), ]) def requireTimePassed(self, filenames): """ Create a replacement for listdir() which only fires after a certain amount of time. """ self.calls = [] def thunk(dirname): now = self.clock.seconds() if now < 20.0: self.calls.append(now) raise OSError(EINVAL, "Not enough time has passed yet.") else: return filenames self.cfp._listdir = thunk def assertRequiredTimePassed(self): """ Assert that calls to the simulated time.sleep() installed by C{requireTimePassed} have been invoked the required number of times. """ # Waiting should be growing by *2 each time until the additional wait # exceeds BACKOFF_MAX (5), at which point we should wait for 5s each # time. def cumulative(values): current = 0.0 for value in values: current += value yield current self.assertEquals( self.calls, list(cumulative([0.0, 0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 5.0, 5.0]))) def test_backoff(self): """ L{CachingFilePath} will wait for an increasing interval up to C{BACKOFF_MAX} between calls to listdir(). """ self.requireTimePassed(['a', 'b', 'c']) self.assertEquals(self.cfp.listdir(), ['a', 'b', 'c']) def test_siblingExtensionSearch(self): """ L{FilePath.siblingExtensionSearch} is unfortunately not implemented in terms of L{FilePath.listdir}, so we need to verify that it will also retry. """ filenames = [ self.cfp.basename() + '.a', self.cfp.basename() + '.b', self.cfp.basename() + '.c' ] siblings = map(self.cfp.sibling, filenames) for sibling in siblings: sibling.touch() self.requireTimePassed(filenames) self.assertEquals(self.cfp.siblingExtensionSearch("*"), siblings[0]) self.assertRequiredTimePassed()