def main(self, args): """ Build all release tarballs. @type args: list of str @param args: The command line arguments to process. This must contain two strings: the checkout directory and the destination directory. """ if len(args) != 2: sys.exit("Must specify two arguments: " "Twisted checkout and destination path") self.buildAllTarballs(FilePath(args[0]), FilePath(args[1]))
def main(self, args): """ Build API documentation. @type args: list of str @param args: The command line arguments to process. This must contain two strings: the path to the root of the Twisted checkout, and a path to an output directory. """ if len(args) != 2: sys.exit("Must specify two arguments: " "Twisted checkout and destination path") self.buildAPIDocs(FilePath(args[0]), FilePath(args[1]))
def buildPDF(self, bookPath, inputDirectory, outputPath): """ Build a PDF from the given a LaTeX book document. @type bookPath: L{FilePath} @param bookPath: The location of a LaTeX document defining a book. @type inputDirectory: L{FilePath} @param inputDirectory: The directory which the inputs of the book are relative to. @type outputPath: L{FilePath} @param outputPath: The location to which to write the resulting book. """ if not bookPath.basename().endswith(".tex"): raise ValueError("Book filename must end with .tex") workPath = FilePath(mkdtemp()) try: startDir = os.getcwd() try: os.chdir(inputDirectory.path) texToDVI = [ "latex", "-interaction=nonstopmode", "-output-directory=" + workPath.path, bookPath.path ] # What I tell you three times is true! # The first two invocations of latex on the book file allows it # correctly create page numbers for in-text references. Why this is # the case, I could not tell you. -exarkun for i in range(3): self.run(texToDVI) bookBaseWithoutExtension = bookPath.basename()[:-4] dviPath = workPath.child(bookBaseWithoutExtension + ".dvi") psPath = workPath.child(bookBaseWithoutExtension + ".ps") pdfPath = workPath.child(bookBaseWithoutExtension + ".pdf") self.run([ "dvips", "-o", psPath.path, "-t", "letter", "-Ppdf", dviPath.path ]) self.run(["ps2pdf13", psPath.path, pdfPath.path]) pdfPath.moveTo(outputPath) workPath.remove() finally: os.chdir(startDir) except: workPath.moveTo(bookPath.parent().child(workPath.basename())) raise
def setUp(self): """ Create a temporary directory with a package structure in it. """ self.entry = FilePath(self.mktemp()) self.preTestModules = sys.modules.copy() sys.path.append(self.entry.path) pkg = self.entry.child("twisted_python_versions_package") pkg.makedirs() pkg.child("__init__.py").setContent( "from reqs.twisted.python.versions import Version\n" "version = Version('twisted_python_versions_package', 1, 0, 0)\n") self.svnEntries = pkg.child(".svn") self.svnEntries.makedirs()
def buildAllTarballs(checkout, destination): """ Build complete tarballs (including documentation) for Twisted and all subprojects. This should be called after the version numbers have been updated and NEWS files created. @type checkout: L{FilePath} @param checkout: The SVN working copy from which a pristine source tree will be exported. @type destination: L{FilePath} @param destination: The directory in which tarballs will be placed. @raise UncleanWorkingDirectory: if there are modifications to the working directory of C{checkout}. @raise NotWorkingDirectory: if the checkout path is not an SVN checkout. """ if not checkout.child(".svn").exists(): raise NotWorkingDirectory( "%s does not appear to be an SVN working directory." % (checkout.path, )) if runCommand(["svn", "st", checkout.path]).strip(): raise UncleanWorkingDirectory( "There are local modifications to the SVN checkout in %s." % (checkout.path, )) workPath = FilePath(mkdtemp()) export = workPath.child("export") runCommand(["svn", "export", checkout.path, export.path]) twistedPath = export.child("twisted") version = Project(twistedPath).getVersion() versionString = version.base() apiBaseURL = "http://twistedmatrix.com/documents/%s/api/%%s.html" % ( versionString) if not destination.exists(): destination.createDirectory() db = DistributionBuilder(export, destination, apiBaseURL=apiBaseURL) db.buildCore(versionString) for subproject in twisted_subprojects: if (subproject not in db.blacklist and twistedPath.child(subproject).exists()): db.buildSubProject(subproject, versionString) db.buildTwisted(versionString) workPath.remove()
def buildAllTarballs(checkout, destination): """ Build complete tarballs (including documentation) for Twisted and all subprojects. This should be called after the version numbers have been updated and NEWS files created. @type checkout: L{FilePath} @param checkout: The SVN working copy from which a pristine source tree will be exported. @type destination: L{FilePath} @param destination: The directory in which tarballs will be placed. @raise UncleanWorkingDirectory: if there are modifications to the working directory of C{checkout}. @raise NotWorkingDirectory: if the checkout path is not an SVN checkout. """ if not checkout.child(".svn").exists(): raise NotWorkingDirectory( "%s does not appear to be an SVN working directory." % (checkout.path,)) if runCommand(["svn", "st", checkout.path]).strip(): raise UncleanWorkingDirectory( "There are local modifications to the SVN checkout in %s." % (checkout.path,)) workPath = FilePath(mkdtemp()) export = workPath.child("export") runCommand(["svn", "export", checkout.path, export.path]) twistedPath = export.child("twisted") version = Project(twistedPath).getVersion() versionString = version.base() apiBaseURL = "http://twistedmatrix.com/documents/%s/api/%%s.html" % ( versionString) if not destination.exists(): destination.createDirectory() db = DistributionBuilder(export, destination, apiBaseURL=apiBaseURL) db.buildCore(versionString) for subproject in twisted_subprojects: if (subproject not in db.blacklist and twistedPath.child(subproject).exists()): db.buildSubProject(subproject, versionString) db.buildTwisted(versionString) workPath.remove()
def _findEntryPathString(self, modobj): """ Determine where a given Python module object came from by looking at path entries. """ topPackageObj = modobj while '.' in topPackageObj.__name__: topPackageObj = self.moduleDict['.'.join( topPackageObj.__name__.split('.')[:-1])] if _isPackagePath(FilePath(topPackageObj.__file__)): # if package 'foo' is on sys.path at /a/b/foo, package 'foo's # __file__ will be /a/b/foo/__init__.py, and we are looking for # /a/b here, the path-entry; so go up two steps. rval = dirname(dirname(topPackageObj.__file__)) else: # the module is completely top-level, not within any packages. The # path entry it's on is just its dirname. rval = dirname(topPackageObj.__file__) # There are probably some awful tricks that an importer could pull # which would break this, so let's just make sure... it's a loaded # module after all, which means that its path MUST be in # path_importer_cache according to PEP 302 -glyph if rval not in self.importerCache: warnings.warn( "%s (for module %s) not in path importer cache " "(PEP 302 violation - check your local configuration)." % (rval, modobj.__name__), stacklevel=3) return rval
def test_getScriptsTopLevel(self): """ Passing the empty string to getScripts returns scripts that are (only) in the top level bin directory. """ basedir = FilePath(self.mktemp()) basedir.createDirectory() bindir = basedir.child("bin") bindir.createDirectory() included = bindir.child("included") included.setContent("yay included") subdir = bindir.child("subdir") subdir.createDirectory() subdir.child("not-included").setContent("not included") scripts = dist.getScripts("", basedir=basedir.path) self.assertEquals(scripts, [included.path])
def buildPDF(self, bookPath, inputDirectory, outputPath): """ Build a PDF from the given a LaTeX book document. @type bookPath: L{FilePath} @param bookPath: The location of a LaTeX document defining a book. @type inputDirectory: L{FilePath} @param inputDirectory: The directory which the inputs of the book are relative to. @type outputPath: L{FilePath} @param outputPath: The location to which to write the resulting book. """ if not bookPath.basename().endswith(".tex"): raise ValueError("Book filename must end with .tex") workPath = FilePath(mkdtemp()) try: startDir = os.getcwd() try: os.chdir(inputDirectory.path) texToDVI = [ "latex", "-interaction=nonstopmode", "-output-directory=" + workPath.path, bookPath.path] # What I tell you three times is true! # The first two invocations of latex on the book file allows it # correctly create page numbers for in-text references. Why this is # the case, I could not tell you. -exarkun for i in range(3): self.run(texToDVI) bookBaseWithoutExtension = bookPath.basename()[:-4] dviPath = workPath.child(bookBaseWithoutExtension + ".dvi") psPath = workPath.child(bookBaseWithoutExtension + ".ps") pdfPath = workPath.child(bookBaseWithoutExtension + ".pdf") self.run([ "dvips", "-o", psPath.path, "-t", "letter", "-Ppdf", dviPath.path]) self.run(["ps2pdf13", psPath.path, pdfPath.path]) pdfPath.moveTo(outputPath) workPath.remove() finally: os.chdir(startDir) except: workPath.moveTo(bookPath.parent().child(workPath.basename())) raise
def mapPath(self, fsPathString): """ Map the given FS path to a ZipPath, by looking at the ZipImporter's "archive" attribute and using it as our ZipArchive root, then walking down into the archive from there. @return: a L{zippath.ZipPath} or L{zippath.ZipArchive} instance. """ za = ZipArchive(self.importer.archive) myPath = FilePath(self.importer.archive) itsPath = FilePath(fsPathString) if myPath == itsPath: return za # This is NOT a general-purpose rule for sys.path or __file__: # zipimport specifically uses regular OS path syntax in its pathnames, # even though zip files specify that slashes are always the separator, # regardless of platform. segs = itsPath.segmentsFrom(myPath) zp = za for seg in segs: zp = zp.child(seg) return zp
def main(self, args): """ Build all news files. @param args: The command line arguments to process. This must contain one string, the path to the base of the Twisted checkout for which to build the news. @type args: C{list} of C{str} """ if len(args) != 1: sys.exit( "Must specify one argument: the path to the Twisted checkout") self.buildAll(FilePath(args[0]))
def test_handshakeFailure(self): """ L{TLSMemoryBIOProtocol} reports errors in the handshake process to the application-level protocol object using its C{connectionLost} method and disconnects the underlying transport. """ clientConnectionLost = Deferred() clientFactory = ClientFactory() clientFactory.protocol = ( lambda: ConnectionLostNotifyingProtocol(clientConnectionLost)) clientContextFactory = HandshakeCallbackContextFactory() wrapperFactory = TLSMemoryBIOFactory(clientContextFactory, True, clientFactory) sslClientProtocol = wrapperFactory.buildProtocol(None) serverConnectionLost = Deferred() serverFactory = ServerFactory() serverFactory.protocol = ( lambda: ConnectionLostNotifyingProtocol(serverConnectionLost)) # This context factory rejects any clients which do not present a # certificate. certificateData = FilePath(certPath).getContent() certificate = PrivateCertificate.loadPEM(certificateData) serverContextFactory = certificate.options(certificate) wrapperFactory = TLSMemoryBIOFactory(serverContextFactory, False, serverFactory) sslServerProtocol = wrapperFactory.buildProtocol(None) connectionDeferred = loopbackAsync(sslServerProtocol, sslClientProtocol) def cbConnectionLost(protocol): # The connection should close on its own in response to the error # induced by the client not supplying the required certificate. # After that, check to make sure the protocol's connectionLost was # called with the right thing. protocol.lostConnectionReason.trap(Error) clientConnectionLost.addCallback(cbConnectionLost) serverConnectionLost.addCallback(cbConnectionLost) # Additionally, the underlying transport should have been told to # go away. return gatherResults( [clientConnectionLost, serverConnectionLost, connectionDeferred])
def main(self, args): """ Given a list of command-line arguments, change all the Twisted versions in the current directory. @type args: list of str @param args: List of command line arguments. This should only contain the version number. """ version_format = ( "Version should be in a form kind of like '1.2.3[pre4]'") if len(args) != 1: sys.exit("Must specify exactly one argument to change-versions") version = args[0] try: major, minor, micro_and_pre = version.split(".") except ValueError: raise SystemExit(version_format) if "pre" in micro_and_pre: micro, pre = micro_and_pre.split("pre") else: micro = micro_and_pre pre = None try: major = int(major) minor = int(minor) micro = int(micro) if pre is not None: pre = int(pre) except ValueError: raise SystemExit(version_format) version_template = Version("Whatever", major, minor, micro, prerelease=pre) self.changeAllProjectVersions(FilePath("."), version_template)
def getStatusChangeTime(self): """ Return the archive file's status change time. """ return FilePath(self.zipfile.filename).getStatusChangeTime()
def getModificationTime(self): """ Return the archive file's modification time. """ return FilePath(self.zipfile.filename).getModificationTime()
def getAccessTime(self): """ Return the archive file's last access time. """ return FilePath(self.zipfile.filename).getAccessTime()
def exists(self): """ Returns true if the underlying archive exists. """ return FilePath(self.zipfile.filename).exists()
def mapPath(self, fsPathString): return FilePath(fsPathString)
class FormatDiscoveryTests(unittest.TestCase): """ Tests which discover the parsing method based on the imported module name. """ def setUp(self): """ Create a temporary directory with a package structure in it. """ self.entry = FilePath(self.mktemp()) self.preTestModules = sys.modules.copy() sys.path.append(self.entry.path) pkg = self.entry.child("twisted_python_versions_package") pkg.makedirs() pkg.child("__init__.py").setContent( "from reqs.twisted.python.versions import Version\n" "version = Version('twisted_python_versions_package', 1, 0, 0)\n") self.svnEntries = pkg.child(".svn") self.svnEntries.makedirs() def tearDown(self): """ Remove the imported modules and sys.path modifications. """ sys.modules.clear() sys.modules.update(self.preTestModules) sys.path.remove(self.entry.path) def checkSVNFormat(self, formatVersion, entriesText, expectedRevision): """ Check for the given revision being detected after setting the SVN entries text and format version of the test directory structure. """ self.svnEntries.child("format").setContent(formatVersion + "\n") self.svnEntries.child("entries").setContent(entriesText) self.assertEqual(self.getVersion()._getSVNVersion(), expectedRevision) def getVersion(self): """ Import and retrieve the Version object from our dynamically created package. """ import twisted_python_versions_package return twisted_python_versions_package.version def test_detectVersion4(self): """ Verify that version 4 format file will be properly detected and parsed. """ self.checkSVNFormat("4", VERSION_4_ENTRIES, '18211') def test_detectVersion8(self): """ Verify that version 8 format files will be properly detected and parsed. """ self.checkSVNFormat("8", VERSION_8_ENTRIES, '22715') def test_detectVersion9(self): """ Verify that version 9 format files will be properly detected and parsed. """ self.checkSVNFormat("9", VERSION_9_ENTRIES, '22715') def test_detectVersion10(self): """ Verify that version 10 format files will be properly detected and parsed. Differing from previous formats, the version 10 format lacks a I{format} file and B{only} has the version information on the first line of the I{entries} file. """ self.svnEntries.child("entries").setContent(VERSION_10_ENTRIES) self.assertEquals(self.getVersion()._getSVNVersion(), '22715') def test_detectUnknownVersion(self): """ Verify that a new version of SVN will result in the revision 'Unknown'. """ self.checkSVNFormat("some-random-new-version", "ooga booga!", 'Unknown')
class ProcessTestsBuilder(ProcessTestsBuilderBase): """ Builder defining tests relating to L{IReactorProcess} for child processes which do not have a PTY. """ usePTY = False keepStdioOpenProgram = FilePath(__file__).sibling('process_helper.py').path if platform.isWindows(): keepStdioOpenArg = "windows" else: # Just a value that doesn't equal "windows" keepStdioOpenArg = "" # Define this test here because PTY-using processes only have stdin and # stdout and the test would need to be different for that to work. def test_childConnectionLost(self): """ L{IProcessProtocol.childConnectionLost} is called each time a file descriptor associated with a child process is closed. """ connected = Deferred() lost = {0: Deferred(), 1: Deferred(), 2: Deferred()} class Closer(ProcessProtocol): def makeConnection(self, transport): connected.callback(transport) def childConnectionLost(self, childFD): lost[childFD].callback(None) source = ("import os, sys\n" "while 1:\n" " line = sys.stdin.readline().strip()\n" " if not line:\n" " break\n" " os.close(int(line))\n") reactor = self.buildReactor() reactor.callWhenRunning(reactor.spawnProcess, Closer(), sys.executable, [sys.executable, "-c", source], usePTY=self.usePTY) def cbConnected(transport): transport.write('2\n') return lost[2].addCallback(lambda ign: transport) connected.addCallback(cbConnected) def lostSecond(transport): transport.write('1\n') return lost[1].addCallback(lambda ign: transport) connected.addCallback(lostSecond) def lostFirst(transport): transport.write('\n') connected.addCallback(lostFirst) connected.addErrback(err) def cbEnded(ignored): reactor.stop() connected.addCallback(cbEnded) self.runReactor(reactor) # This test is here because PTYProcess never delivers childConnectionLost. def test_processEnded(self): """ L{IProcessProtocol.processEnded} is called after the child process exits and L{IProcessProtocol.childConnectionLost} is called for each of its file descriptors. """ ended = Deferred() lost = [] class Ender(ProcessProtocol): def childDataReceived(self, fd, data): msg('childDataReceived(%d, %r)' % (fd, data)) self.transport.loseConnection() def childConnectionLost(self, childFD): msg('childConnectionLost(%d)' % (childFD, )) lost.append(childFD) def processExited(self, reason): msg('processExited(%r)' % (reason, )) def processEnded(self, reason): msg('processEnded(%r)' % (reason, )) ended.callback([reason]) reactor = self.buildReactor() reactor.callWhenRunning(reactor.spawnProcess, Ender(), sys.executable, [ sys.executable, self.keepStdioOpenProgram, "child", self.keepStdioOpenArg ], usePTY=self.usePTY) def cbEnded((failure, )): failure.trap(ProcessDone) self.assertEqual(set(lost), set([0, 1, 2])) ended.addCallback(cbEnded) ended.addErrback(err) ended.addCallback(lambda ign: reactor.stop()) self.runReactor(reactor) # This test is here because PTYProcess.loseConnection does not actually # close the file descriptors to the child process. This test needs to be # written fairly differently for PTYProcess. def test_processExited(self): """ L{IProcessProtocol.processExited} is called when the child process exits, even if file descriptors associated with the child are still open. """ exited = Deferred() allLost = Deferred() lost = [] class Waiter(ProcessProtocol): def childDataReceived(self, fd, data): msg('childDataReceived(%d, %r)' % (fd, data)) def childConnectionLost(self, childFD): msg('childConnectionLost(%d)' % (childFD, )) lost.append(childFD) if len(lost) == 3: allLost.callback(None) def processExited(self, reason): msg('processExited(%r)' % (reason, )) # See test_processExitedWithSignal exited.callback([reason]) self.transport.loseConnection() reactor = self.buildReactor() reactor.callWhenRunning(reactor.spawnProcess, Waiter(), sys.executable, [ sys.executable, self.keepStdioOpenProgram, "child", self.keepStdioOpenArg ], usePTY=self.usePTY) def cbExited((failure, )): failure.trap(ProcessDone) msg('cbExited; lost = %s' % (lost, )) self.assertEqual(lost, []) return allLost exited.addCallback(cbExited) def cbAllLost(ignored): self.assertEqual(set(lost), set([0, 1, 2])) exited.addCallback(cbAllLost) exited.addErrback(err) exited.addCallback(lambda ign: reactor.stop()) self.runReactor(reactor) def makeSourceFile(self, sourceLines): """ Write the given list of lines to a text file and return the absolute path to it. """ script = self.mktemp() scriptFile = file(script, 'wt') scriptFile.write(os.linesep.join(sourceLines) + os.linesep) scriptFile.close() return os.path.abspath(script) def test_shebang(self): """ Spawning a process with an executable which is a script starting with an interpreter definition line (#!) uses that interpreter to evaluate the script. """ SHEBANG_OUTPUT = 'this is the shebang output' scriptFile = self.makeSourceFile([ "#!%s" % (sys.executable, ), "import sys", "sys.stdout.write('%s')" % (SHEBANG_OUTPUT, ), "sys.stdout.flush()" ]) os.chmod(scriptFile, 0700) reactor = self.buildReactor() def cbProcessExited((out, err, code)): msg("cbProcessExited((%r, %r, %d))" % (out, err, code)) self.assertEqual(out, SHEBANG_OUTPUT) self.assertEqual(err, "") self.assertEqual(code, 0) def shutdown(passthrough): reactor.stop() return passthrough def start(): d = utils.getProcessOutputAndValue(scriptFile, reactor=reactor) d.addBoth(shutdown) d.addCallback(cbProcessExited) d.addErrback(err) reactor.callWhenRunning(start) self.runReactor(reactor) def test_processCommandLineArguments(self): """ Arguments given to spawnProcess are passed to the child process as originally intended. """ source = ( # On Windows, stdout is not opened in binary mode by default, # so newline characters are munged on writing, interfering with # the tests. 'import sys, os\n' 'try:\n' ' import msvcrt\n' ' msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)\n' 'except ImportError:\n' ' pass\n' 'for arg in sys.argv[1:]:\n' ' sys.stdout.write(arg + chr(0))\n' ' sys.stdout.flush()') args = ['hello', '"', ' \t|<>^&', r'"\\"hello\\"', r'"foo\ bar baz\""'] # Ensure that all non-NUL characters can be passed too. args.append(''.join(map(chr, xrange(1, 256)))) reactor = self.buildReactor() def processFinished(output): output = output.split('\0') # Drop the trailing \0. output.pop() self.assertEquals(args, output) def shutdown(result): reactor.stop() return result def spawnChild(): d = succeed(None) d.addCallback(lambda dummy: utils.getProcessOutput( sys.executable, ['-c', source] + args, reactor=reactor)) d.addCallback(processFinished) d.addBoth(shutdown) reactor.callWhenRunning(spawnChild) self.runReactor(reactor)
class FormatDiscoveryTests(unittest.TestCase): """ Tests which discover the parsing method based on the imported module name. """ def setUp(self): """ Create a temporary directory with a package structure in it. """ self.entry = FilePath(self.mktemp()) self.preTestModules = sys.modules.copy() sys.path.append(self.entry.path) pkg = self.entry.child("twisted_python_versions_package") pkg.makedirs() pkg.child("__init__.py").setContent( "from reqs.twisted.python.versions import Version\n" "version = Version('twisted_python_versions_package', 1, 0, 0)\n") self.svnEntries = pkg.child(".svn") self.svnEntries.makedirs() def tearDown(self): """ Remove the imported modules and sys.path modifications. """ sys.modules.clear() sys.modules.update(self.preTestModules) sys.path.remove(self.entry.path) def checkSVNFormat(self, formatVersion, entriesText, expectedRevision): """ Check for the given revision being detected after setting the SVN entries text and format version of the test directory structure. """ self.svnEntries.child("format").setContent(formatVersion+"\n") self.svnEntries.child("entries").setContent(entriesText) self.assertEqual(self.getVersion()._getSVNVersion(), expectedRevision) def getVersion(self): """ Import and retrieve the Version object from our dynamically created package. """ import twisted_python_versions_package return twisted_python_versions_package.version def test_detectVersion4(self): """ Verify that version 4 format file will be properly detected and parsed. """ self.checkSVNFormat("4", VERSION_4_ENTRIES, '18211') def test_detectVersion8(self): """ Verify that version 8 format files will be properly detected and parsed. """ self.checkSVNFormat("8", VERSION_8_ENTRIES, '22715') def test_detectVersion9(self): """ Verify that version 9 format files will be properly detected and parsed. """ self.checkSVNFormat("9", VERSION_9_ENTRIES, '22715') def test_detectVersion10(self): """ Verify that version 10 format files will be properly detected and parsed. Differing from previous formats, the version 10 format lacks a I{format} file and B{only} has the version information on the first line of the I{entries} file. """ self.svnEntries.child("entries").setContent(VERSION_10_ENTRIES) self.assertEquals(self.getVersion()._getSVNVersion(), '22715') def test_detectUnknownVersion(self): """ Verify that a new version of SVN will result in the revision 'Unknown'. """ self.checkSVNFormat("some-random-new-version", "ooga booga!", 'Unknown')