def test_file_logging_rotation_5_files(self):
"""
Only 5 logfiles are kept.
"""
logfile = FilePath(self.mktemp()).child('foo.log')
logfile.parent().makedirs()
# This file will become foo.log.1
with logfile.open('w') as f:
f.write(b'0')
f.truncate(int(MiB(100).to_Byte().value))
# These file extensions will be incremented
for i in range(1, 5):
sibling = logfile.sibling(logfile.basename() + u'.' + unicode(i))
with sibling.open('w') as f:
f.write(bytes(i))
d = self.run_script(EliotScript, options=['--logfile', logfile.path])
def verify_logfiles(stdout_messages, logfile):
logfile_dir = logfile.parent()
self.assertEqual(
# The contents of the files will now be an integer one less
# than the integer in the file name.
map(bytes, range(0, 4)),
list(
logfile_dir.child('foo.log.{}'.format(i)).open().read(1)
for i in range(1, 5)))
d.addCallback(verify_logfiles, logfile=logfile)
return d
def test_file_logging_rotation_5_files(self):
"""
Only 5 logfiles are kept.
"""
logfile = FilePath(self.mktemp()).child('foo.log')
logfile.parent().makedirs()
# This file will become foo.log.1
with logfile.open('w') as f:
f.write(b'0')
f.truncate(int(MiB(100).to_Byte().value))
# These file extensions will be incremented
for i in range(1, 5):
sibling = logfile.sibling(logfile.basename() + u'.' + unicode(i))
with sibling.open('w') as f:
f.write(bytes(i))
d = self.run_script(EliotScript, options=['--logfile', logfile.path])
def verify_logfiles(stdout_messages, logfile):
logfile_dir = logfile.parent()
self.assertEqual(
# The contents of the files will now be an integer one less
# than the integer in the file name.
map(bytes, range(0, 4)),
list(
logfile_dir.child('foo.log.{}'.format(i)).open().read(1)
for i
in range(1, 5)
)
)
d.addCallback(verify_logfiles, logfile=logfile)
return d
def test_missingSource(self):
"""
If a function the source of which is not available is including in the
C{offendingFunctions} list, L{TestCase.flushWarnings} raises
L{IOError}. Such a call flushes no warnings.
"""
package = FilePath(self.mktemp()).child('twisted_private_helper')
package.makedirs()
package.child('__init__.py').setContent('')
package.child('missingsourcefile.py').setContent('''
import warnings
def foo():
warnings.warn("oh no")
''')
sys.path.insert(0, package.parent().path)
self.addCleanup(sys.path.remove, package.parent().path)
from twisted_private_helper import missingsourcefile
self.addCleanup(sys.modules.pop, 'twisted_private_helper')
self.addCleanup(sys.modules.pop, missingsourcefile.__name__)
package.child('missingsourcefile.py').remove()
missingsourcefile.foo()
self.assertRaises(
IOError, self.flushWarnings, [missingsourcefile.foo])
self.assertEqual(len(self.flushWarnings()), 1)
def test_file_logging(self):
"""
Logged messages are written to ``logfile`` if ``--logfile`` is supplied
on the command line.
"""
logfile = FilePath(self.mktemp()).child('foo.log')
logfile.parent().makedirs()
d = self.run_script(EliotScript, options=['--logfile', logfile.path])
d.addCallback(self._assert_logfile_messages, logfile=logfile)
return d
def test_file_logging(self):
"""
Logged messages are written to ``logfile`` if ``--logfile`` is supplied
on the command line.
"""
logfile = FilePath(self.mktemp()).child('foo.log')
logfile.parent().makedirs()
d = self.run_script(EliotScript, options=['--logfile', logfile.path])
d.addCallback(self._assert_logfile_messages, logfile=logfile)
return d
def test_mktemp_doesnt_exist(self, test):
"""
``mktemp`` returns a path that doesn't exist inside a directory that
does.
"""
temp_path = FilePath(test.mktemp())
self.addCleanup(_remove_dir, temp_path.parent())
self.expectThat(temp_path.parent().path, DirExists())
self.expectThat(temp_path.path, Not(PathExists()))
self.assertThat(temp_path, BelowPath(FilePath(os.getcwd())))
def test_mktemp_doesnt_exist(self, test):
"""
``mktemp`` returns a path that doesn't exist inside a directory that
does.
"""
temp_path = FilePath(test.mktemp())
self.addCleanup(_remove_dir, temp_path.parent())
self.expectThat(temp_path.parent().path, DirExists())
self.expectThat(temp_path.path, Not(PathExists()))
self.assertThat(temp_path, BelowPath(FilePath(os.getcwd())))
def get_client(options):
cluster = FilePath(options["cluster-yml"])
if cluster.exists():
config = yaml.load(cluster.open())
certificates_path = cluster.parent()
user = config["users"][0]
control_service = None # figure it out based on cluster.yml
else:
certificates_path = FilePath(options["certs-path"])
if options["user"] is None:
raise UsageError("must specify --user")
user = options["user"]
if options["control-service"] is None:
raise UsageError("must specify --control-service")
control_service = options["control-service"]
user_certificate_filename = "%s.crt" % (user, )
user_key_filename = "%s.key" % (user, )
return txflocker_get_client(
certificates_path=certificates_path,
user_certificate_filename=user_certificate_filename,
user_key_filename=user_key_filename,
target_hostname=control_service,
)
def test_missingSource(self):
"""
Warnings emitted by a function the source code of which is not
available can still be flushed.
"""
package = FilePath(self.mktemp().encode("utf-8")).child(
b"twisted_private_helper"
)
package.makedirs()
package.child(b"__init__.py").setContent(b"")
package.child(b"missingsourcefile.py").setContent(
b"""
import warnings
def foo():
warnings.warn("oh no")
"""
)
pathEntry = package.parent().path.decode("utf-8")
sys.path.insert(0, pathEntry)
self.addCleanup(sys.path.remove, pathEntry)
from twisted_private_helper import missingsourcefile # type: ignore[import]
self.addCleanup(sys.modules.pop, "twisted_private_helper")
self.addCleanup(sys.modules.pop, missingsourcefile.__name__)
package.child(b"missingsourcefile.py").remove()
missingsourcefile.foo()
self.assertEqual(len(self.flushWarnings([missingsourcefile.foo])), 1)
def test_mktemp_doesnt_exist(self, base_test_case):
"""
``mktemp`` returns a path that doesn't exist inside a directory that
does.
"""
class SomeTest(base_test_case):
def test_pass(self):
pass
test = SomeTest('test_pass')
temp_path = FilePath(test.mktemp())
self.addCleanup(_remove_dir, temp_path.parent())
self.expectThat(temp_path.parent().path, DirExists())
self.expectThat(temp_path.path, Not(PathExists()))
self.assertThat(temp_path, BelowPath(FilePath(os.getcwd())))
def test_rewriteCss(self):
"""
Test that CSS processing works, and verify the header.
"""
clock = Clock()
fc = FileCache(lambda: clock.seconds(), 1)
temp = FilePath(self.mktemp() + '.css')
with temp.open('wb') as f:
f.write("p { color: red; }\n")
# BetterFile(temp.path) would not work because the processing happens
# in getChild. So, create a BetterFile for the .css file's parent dir.
bf = BetterFile(temp.parent().path, fileCache=fc, rewriteCss=True)
d = self._requestPostpathAndRender(bf, [temp.basename()])
headerRe = re.compile(r"/\* CSSResource processed ([0-9a-f]{32}?) \*/")
def assertProcessedContent((request, child)):
out = "".join(request.written)
lines = out.split("\n")
self.assertTrue(re.match(headerRe, lines[0]), lines[0])
self.assertEqual("p { color: red; }", lines[1])
self.assertEqual("", lines[2])
self.assertEqual(3, len(lines))
d.addCallback(assertProcessedContent)
return d
def get_client(options):
cluster = FilePath(options["cluster-yml"])
if cluster.exists():
config = yaml.load(cluster.open())
certificates_path = cluster.parent()
user = config["users"][0]
control_service = None # figure it out based on cluster.yml
else:
certificates_path = FilePath(options["certs-path"])
if options["user"] is None:
raise UsageError("must specify --user")
user = options["user"]
if options["control-service"] is None:
raise UsageError("must specify --control-service")
control_service = options["control-service"]
user_certificate_filename = "%s.crt" % (user,)
user_key_filename = "%s.key" % (user,)
return txflocker_get_client(
certificates_path=certificates_path,
user_certificate_filename=user_certificate_filename,
user_key_filename=user_key_filename,
target_hostname=control_service,
)
def test_renamedSource(self):
"""
Warnings emitted by a function defined in a file which has been renamed
since it was initially compiled can still be flushed.
This is testing the code which specifically supports working around the
unfortunate behavior of CPython to write a .py source file name into
the .pyc files it generates and then trust that it is correct in
various places. If source files are renamed, .pyc files may not be
regenerated, but they will contain incorrect filenames.
"""
package = FilePath(self.mktemp().encode("utf-8")).child(
b"twisted_private_helper"
)
package.makedirs()
package.child(b"__init__.py").setContent(b"")
package.child(b"module.py").setContent(
b"""
import warnings
def foo():
warnings.warn("oh no")
"""
)
pathEntry = package.parent().path.decode("utf-8")
sys.path.insert(0, pathEntry)
self.addCleanup(sys.path.remove, pathEntry)
# Import it to cause pycs to be generated
from twisted_private_helper import module
# Clean up the state resulting from that import; we're not going to use
# this module, so it should go away.
del sys.modules["twisted_private_helper"]
del sys.modules[module.__name__]
# Some Python versions have extra state related to the just
# imported/renamed package. Clean it up too. See also
# http://bugs.python.org/issue15912
try:
from importlib import invalidate_caches
except ImportError:
pass
else:
invalidate_caches()
# Rename the source directory
package.moveTo(package.sibling(b"twisted_renamed_helper"))
# Import the newly renamed version
from twisted_renamed_helper import module # type: ignore[import]
self.addCleanup(sys.modules.pop, "twisted_renamed_helper")
self.addCleanup(sys.modules.pop, module.__name__)
# Generate the warning
module.foo()
# Flush it
self.assertEqual(len(self.flushWarnings([module.foo])), 1)
def test_reactorSelection(self):
"""
L{AxiomaticStart} optionally takes the name of a reactor and
installs it instead of the default reactor.
"""
# Since this process is already hopelessly distant from the state in
# which I{axiomatic start} operates, it would make no sense to try a
# functional test of this behavior in this process. Since the
# behavior being tested involves lots of subtle interactions between
# lots of different pieces of code (the reactor might get installed
# at the end of a ten-deep chain of imports going through as many
# different projects), it also makes no sense to try to make this a
# unit test. So, start a child process and try to use the alternate
# reactor functionality there.
here = FilePath(__file__)
# Try to find it relative to the source of this test.
bin = here.parent().parent().parent().child("bin")
axiomatic = bin.child("axiomatic")
if axiomatic.exists():
# Great, use that one.
axiomatic = axiomatic.path
else:
# Try to find it on the path, instead.
axiomatics = which("axiomatic")
if axiomatics:
# Great, it was on the path.
axiomatic = axiomatics[0]
else:
# Nope, not there, give up.
raise SkipTest(
"Could not find axiomatic script on path or at %s" % (
axiomatic.path,))
# Create a store for the child process to use and put an item in it.
# This will force an import of the module that defines that item's
# class when the child process starts. The module imports the default
# reactor at the top-level, making this the worst-case for the reactor
# selection code.
storePath = self.mktemp()
store = Store(storePath)
SomeItem(store=store)
store.close()
# Install select reactor because it available on all platforms, and
# it is still an error to try to install the select reactor even if
# the already installed reactor was the select reactor.
argv = [
sys.executable,
axiomatic, "-d", storePath,
"start", "--reactor", "select", "-n"]
expected = [
"reactor class: twisted.internet.selectreactor.SelectReactor.",
"reactor class: <class 'twisted.internet.selectreactor.SelectReactor'>"]
proto, complete = AxiomaticStartProcessProtocol.protocolAndDeferred(expected)
environ = os.environ.copy()
reactor.spawnProcess(proto, sys.executable, argv, env=environ)
return complete
def test_noKnownHostsOption(self):
"""
L{default.verifyHostKey} should find your known_hosts file in
~/.ssh/known_hosts if you don't specify one explicitly on the command
line.
"""
l = []
tmpdir = self.mktemp()
oldHostsOption = self.hostsOption
hostsNonOption = FilePath(tmpdir).child(".ssh").child("known_hosts")
hostsNonOption.parent().makedirs()
FilePath(oldHostsOption).moveTo(hostsNonOption)
self.replaceHome(tmpdir)
self.options['known-hosts'] = None
default.verifyHostKey(self.fakeTransport, "4.3.2.1", sampleKey,
"I don't care.").addCallback(l.append)
self.assertEqual([1], l)
def test_mktemp_doesnt_exist(self, base_test_case):
"""
``mktemp`` returns a path that doesn't exist inside a directory that
does.
"""
class SomeTest(base_test_case):
def test_pass(self):
pass
test = SomeTest('test_pass')
temp_path = FilePath(test.mktemp())
self.addCleanup(_remove_dir, temp_path.parent())
self.expectThat(temp_path.parent().path, DirExists())
self.expectThat(temp_path.path, Not(PathExists()))
self.assertThat(temp_path, BelowPath(FilePath(os.getcwd())))
def test_noKnownHostsOption(self):
"""
L{default.verifyHostKey} should find your known_hosts file in
~/.ssh/known_hosts if you don't specify one explicitly on the command
line.
"""
l = []
tmpdir = self.mktemp()
oldHostsOption = self.hostsOption
hostsNonOption = FilePath(tmpdir).child(".ssh").child("known_hosts")
hostsNonOption.parent().makedirs()
FilePath(oldHostsOption).moveTo(hostsNonOption)
self.replaceHome(tmpdir)
self.options['known-hosts'] = None
default.verifyHostKey(self.fakeTransport, "4.3.2.1", sampleKey,
"I don't care.").addCallback(l.append)
self.assertEqual([1], l)
def test_renamedSource(self):
"""
Warnings emitted by a function defined in a file which has been renamed
since it was initially compiled can still be flushed.
This is testing the code which specifically supports working around the
unfortunate behavior of CPython to write a .py source file name into
the .pyc files it generates and then trust that it is correct in
various places. If source files are renamed, .pyc files may not be
regenerated, but they will contain incorrect filenames.
"""
package = FilePath(self.mktemp()).child('twisted_private_helper')
package.makedirs()
package.child('__init__.py').setContent('')
package.child('module.py').setContent('''
import warnings
def foo():
warnings.warn("oh no")
''')
sys.path.insert(0, package.parent().path)
self.addCleanup(sys.path.remove, package.parent().path)
# Import it to cause pycs to be generated
from twisted_private_helper import module
# Clean up the state resulting from that import; we're not going to use
# this module, so it should go away.
del sys.modules['twisted_private_helper']
del sys.modules[module.__name__]
# Rename the source directory
package.moveTo(package.sibling('twisted_renamed_helper'))
# Import the newly renamed version
from twisted_renamed_helper import module
self.addCleanup(sys.modules.pop, 'twisted_renamed_helper')
self.addCleanup(sys.modules.pop, module.__name__)
# Generate the warning
module.foo()
# Flush it
self.assertEqual(len(self.flushWarnings([module.foo])), 1)
def test_file_logging_rotation_at_100MiB(self):
"""
Logfiles are rotated when they reach 100MiB.
"""
logfile = FilePath(self.mktemp()).child('foo.log')
logfile.parent().makedirs()
with logfile.open('w') as f:
f.truncate(int(MiB(100).to_Byte().value - 1))
d = self.run_script(EliotScript, options=['--logfile', logfile.path])
def verify_logfiles(stdout_messages, logfile):
self.assertEqual(
set([logfile, logfile.sibling(logfile.basename() + u'.1')]),
set(logfile.parent().children())
)
d.addCallback(verify_logfiles, logfile=logfile)
return d
def test_renamedSource(self):
"""
Warnings emitted by a function defined in a file which has been renamed
since it was initially compiled can still be flushed.
This is testing the code which specifically supports working around the
unfortunate behavior of CPython to write a .py source file name into
the .pyc files it generates and then trust that it is correct in
various places. If source files are renamed, .pyc files may not be
regenerated, but they will contain incorrect filenames.
"""
package = FilePath(self.mktemp()).child('twisted_private_helper')
package.makedirs()
package.child('__init__.py').setContent('')
package.child('module.py').setContent('''
import warnings
def foo():
warnings.warn("oh no")
''')
sys.path.insert(0, package.parent().path)
self.addCleanup(sys.path.remove, package.parent().path)
# Import it to cause pycs to be generated
from twisted_private_helper import module
# Clean up the state resulting from that import; we're not going to use
# this module, so it should go away.
del sys.modules['twisted_private_helper']
del sys.modules[module.__name__]
# Rename the source directory
package.moveTo(package.sibling('twisted_renamed_helper'))
# Import the newly renamed version
from twisted_renamed_helper import module
self.addCleanup(sys.modules.pop, 'twisted_renamed_helper')
self.addCleanup(sys.modules.pop, module.__name__)
# Generate the warning
module.foo()
# Flush it
self.assertEqual(len(self.flushWarnings([module.foo])), 1)
def test_file_logging_rotation_at_100MiB(self):
"""
Logfiles are rotated when they reach 100MiB.
"""
logfile = FilePath(self.mktemp()).child('foo.log')
logfile.parent().makedirs()
with logfile.open('w') as f:
f.truncate(int(MiB(100).to_Byte().value - 1))
d = self.run_script(EliotScript, options=['--logfile', logfile.path])
def verify_logfiles(stdout_messages, logfile):
self.assertEqual(
set([logfile, logfile.sibling(logfile.basename() + u'.1')]),
set(logfile.parent().children())
)
d.addCallback(verify_logfiles, logfile=logfile)
return d
def _get_cloudformation_full_path():
"""
Get fully qualified pathname of cloudformation.py script.
:returns: Fully qualified pathname of cloudformation.py
:rtype: twisted.python.filepath.FilePath
"""
test_directory_filepath = FilePath(__file__)
installer_directory_filepath = test_directory_filepath.parent().parent()
cloudformation_filepath = installer_directory_filepath.childSearchPreauth(
u'cloudformation.py')
return cloudformation_filepath
def test_warnings_suppressed(self):
"""
Warnings are suppressed for processes that import flocker.
"""
root = FilePath(flocker.__file__)
result = check_output(
[executable, b"-c", (b"import flocker; import warnings; " +
b"warnings.warn('ohno')")],
stderr=STDOUT,
# Make sure we can import flocker package:
cwd=root.parent().parent().path)
self.assertEqual(result, b"")
def _get_cloudformation_full_path():
"""
Get fully qualified pathname of cloudformation.py script.
:returns: Fully qualified pathname of cloudformation.py
:rtype: twisted.python.filepath.FilePath
"""
test_directory_filepath = FilePath(__file__)
installer_directory_filepath = test_directory_filepath.parent().parent()
cloudformation_filepath = installer_directory_filepath.childSearchPreauth(
u'cloudformation.py')
return cloudformation_filepath
def test_missingSource(self):
"""
Warnings emitted by a function the source code of which is not
available can still be flushed.
"""
package = FilePath(self.mktemp()).child('twisted_private_helper')
package.makedirs()
package.child('__init__.py').setContent('')
package.child('missingsourcefile.py').setContent('''
import warnings
def foo():
warnings.warn("oh no")
''')
sys.path.insert(0, package.parent().path)
self.addCleanup(sys.path.remove, package.parent().path)
from twisted_private_helper import missingsourcefile
self.addCleanup(sys.modules.pop, 'twisted_private_helper')
self.addCleanup(sys.modules.pop, missingsourcefile.__name__)
package.child('missingsourcefile.py').remove()
missingsourcefile.foo()
self.assertEqual(len(self.flushWarnings([missingsourcefile.foo])), 1)
def test_missingSource(self):
"""
Warnings emitted by a function the source code of which is not
available can still be flushed.
"""
package = FilePath(self.mktemp()).child('twisted_private_helper')
package.makedirs()
package.child('__init__.py').setContent('')
package.child('missingsourcefile.py').setContent('''
import warnings
def foo():
warnings.warn("oh no")
''')
sys.path.insert(0, package.parent().path)
self.addCleanup(sys.path.remove, package.parent().path)
from twisted_private_helper import missingsourcefile
self.addCleanup(sys.modules.pop, 'twisted_private_helper')
self.addCleanup(sys.modules.pop, missingsourcefile.__name__)
package.child('missingsourcefile.py').remove()
missingsourcefile.foo()
self.assertEqual(len(self.flushWarnings([missingsourcefile.foo])), 1)
def _storeArtifact(cls, content: IO[bytes], path: FilePath) -> None:
"""Verify and store an artifact from a temporary file.
If the file is valid, store it at the given path.
If the file is not valid, raise ValueError.
"""
cls.verify(content)
# Copy content.
content.seek(0, 0)
uploadPath = path.siblingExtension('.part')
path.parent().makedirs(ignoreExistingDirectory=True)
with uploadPath.open('wb') as out:
while True:
data = content.read(cls.putBlockSize)
if not data:
break
out.write(data)
out.flush()
fsync(out.fileno())
replace(uploadPath.path, path.path)
path.changed()
def opt_logfile(self, logfile_path):
"""
Log to a file. Log is written to ``stdout`` by default. The logfile
directory is created if it does not already exist.
"""
logfile = FilePath(logfile_path)
logfile_directory = logfile.parent()
if not logfile_directory.exists():
logfile_directory.makedirs()
self['logfile'] = LogFile.fromFullPath(
logfile.path,
rotateLength=LOGFILE_LENGTH,
maxRotatedFiles=LOGFILE_COUNT,
)
def opt_logfile(self, logfile_path):
"""
Log to a file. Log is written to ``stdout`` by default. The logfile
directory is created if it does not already exist.
"""
logfile = FilePath(logfile_path)
logfile_directory = logfile.parent()
if not logfile_directory.exists():
logfile_directory.makedirs()
self['logfile'] = LogFile.fromFullPath(
logfile.path,
rotateLength=LOGFILE_LENGTH,
maxRotatedFiles=LOGFILE_COUNT,
)
def test_missingSource(self):
"""
If a function the source of which is not available is including in the
C{offendingFunctions} list, L{TestCase.flushWarnings} raises
L{IOError}. Such a call flushes no warnings.
"""
package = FilePath(self.mktemp()).child('twisted_private_helper')
package.makedirs()
package.child('__init__.py').setContent('')
package.child('missingsourcefile.py').setContent('''
import warnings
def foo():
warnings.warn("oh no")
''')
sys.path.insert(0, package.parent().path)
self.addCleanup(sys.path.remove, package.parent().path)
from twisted_private_helper import missingsourcefile
self.addCleanup(sys.modules.pop, 'twisted_private_helper')
self.addCleanup(sys.modules.pop, missingsourcefile.__name__)
package.child('missingsourcefile.py').remove()
missingsourcefile.foo()
self.assertRaises(IOError, self.flushWarnings, [missingsourcefile.foo])
self.assertEqual(len(self.flushWarnings()), 1)
def test_warnings_suppressed(self):
"""
Warnings are suppressed for processes that import flocker.
"""
root = FilePath(flocker.__file__)
result = check_output(
[
executable, b"-c",
(b"import flocker; import warnings; " +
b"warnings.warn('ohno')")
],
stderr=STDOUT,
# Make sure we can import flocker package:
cwd=root.parent().parent().path)
self.assertEqual(result, b"")
def test_openFileDescriptors(self):
"""
A spawned process has only stdin, stdout and stderr open
(file descriptor 3 is also reported as open, because of the call to
'os.listdir()').
"""
from twisted.python.runtime import platformType
if platformType != "posix":
raise SkipTest("Test only applies to POSIX platforms")
here = FilePath(__file__)
top = here.parent().parent().parent().parent()
source = (
"import sys",
"sys.path.insert(0, '%s')" % (top.path,),
"from twisted.internet import process",
"sys.stdout.write(str(process._listOpenFDs()))",
"sys.stdout.flush()",
)
def checkOutput(output):
self.assertEqual("[0, 1, 2, 3]", output)
reactor = self.buildReactor()
class Protocol(ProcessProtocol):
def __init__(self):
self.output = []
def outReceived(self, data):
self.output.append(data)
def processEnded(self, reason):
try:
checkOutput("".join(self.output))
finally:
reactor.stop()
proto = Protocol()
reactor.callWhenRunning(
reactor.spawnProcess,
proto,
sys.executable,
[sys.executable, "-Wignore", "-c", "\n".join(source)],
usePTY=self.usePTY,
)
self.runReactor(reactor)
def run(self):
schema_store = namedAny(self.options["schema_store_fqpn"])
appContainer = namedAny(self.arguments[0])
# This is the path of the file that contains the autoklein directive.
src_path = FilePath(self.state_machine.get_source(self.lineno))
# self.options["examples_path"] is a path relative to the source file
# containing it to a file containing examples to include.
examples_path = src_path.parent().preauthChild(
self.options["examples_path"])
self._examples = _loadExamples(examples_path)
# The contents of the example file are included in the output so the
# example file is a dependency of the document.
self.state.document.settings.record_dependencies.add(
examples_path.path)
# The following three lines record (some?) of the dependencies of the
# directive, so automatic regeneration happens.
# Specifically, it records this file, and the file where the app
# is declared. If we ever have routes for a single app declared
# across multiple files, this will need to be updated.
appFileName = getsourcefile(appContainer)
self.state.document.settings.record_dependencies.add(appFileName)
self.state.document.settings.record_dependencies.add(__file__)
# Copied from sphinxcontrib.autohttp.flask
# Return the result of parsing the rst f
node = nodes.section()
node.document = self.state.document
result = ViewList()
restLines = makeRst(
prefix=self.options['prefix'],
section=self.options['section'],
app=appContainer.app,
exampleByIdentifier=self._exampleByIdentifier,
schema_store=schema_store)
for line in restLines:
result.append(line, '<autoklein>')
nested_parse_with_titles(self.state, result, node)
return node.children
def test_toptobottomMissingSource(self):
"""
--order=toptobottom detects the source line of methods from modules
whose source file is missing.
"""
tempdir = self.mktemp()
package = FilePath(tempdir).child("twisted_toptobottom_temp")
package.makedirs()
package.child("__init__.py").setContent(b"")
package.child("test_missing.py").setContent(
textwrap.dedent(
"""
from twisted.trial.unittest import TestCase
class TestMissing(TestCase):
def test_second(self): pass
def test_third(self): pass
def test_fourth(self): pass
def test_first(self): pass
"""
).encode("utf8")
)
pathEntry = package.parent().path
sys.path.insert(0, pathEntry)
self.addCleanup(sys.path.remove, pathEntry)
from twisted_toptobottom_temp import test_missing
self.addCleanup(sys.modules.pop, "twisted_toptobottom_temp")
self.addCleanup(sys.modules.pop, test_missing.__name__)
package.child("test_missing.py").remove()
self.config.parseOptions(
["--order", "toptobottom", "twisted.trial.test.ordertests"]
)
loader = trial._getLoader(self.config)
suite = loader.loadModule(test_missing)
self.assertEqual(
testNames(suite),
[
"twisted_toptobottom_temp.test_missing.TestMissing.test_second",
"twisted_toptobottom_temp.test_missing.TestMissing.test_third",
"twisted_toptobottom_temp.test_missing.TestMissing.test_fourth",
"twisted_toptobottom_temp.test_missing.TestMissing.test_first",
],
)
def run(self):
schema_store = namedAny(self.options["schema_store_fqpn"])
appContainer = namedAny(self.arguments[0])
# This is the path of the file that contains the autoklein directive.
src_path = FilePath(self.state_machine.get_source(self.lineno))
# self.options["examples_path"] is a path relative to the source file
# containing it to a file containing examples to include.
examples_path = src_path.parent().preauthChild(
self.options["examples_path"])
self._examples = _loadExamples(examples_path)
# The contents of the example file are included in the output so the
# example file is a dependency of the document.
self.state.document.settings.record_dependencies.add(
examples_path.path)
# The following three lines record (some?) of the dependencies of the
# directive, so automatic regeneration happens.
# Specifically, it records this file, and the file where the app
# is declared. If we ever have routes for a single app declared
# across multiple files, this will need to be updated.
appFileName = getsourcefile(appContainer)
self.state.document.settings.record_dependencies.add(appFileName)
self.state.document.settings.record_dependencies.add(__file__)
# Copied from sphinxcontrib.autohttp.flask
# Return the result of parsing the rst f
node = nodes.section()
node.document = self.state.document
result = ViewList()
restLines = makeRst(
prefix=self.options['prefix'],
section=self.options['section'],
app=appContainer.app,
exampleByIdentifier=self._exampleByIdentifier,
schema_store=schema_store)
for line in restLines:
result.append(line, '<autoklein>')
nested_parse_with_titles(self.state, result, node)
return node.children
def test_openFileDescriptors(self):
"""
A spawned process has only stdin, stdout and stderr open
(file descriptor 3 is also reported as open, because of the call to
'os.listdir()').
"""
from twisted.python.runtime import platformType
if platformType != "posix":
raise SkipTest("Test only applies to POSIX platforms")
here = FilePath(__file__)
top = here.parent().parent().parent().parent()
source = (
"import sys",
"sys.path.insert(0, '%s')" % (top.path,),
"from twisted.internet import process",
"sys.stdout.write(str(process._listOpenFDs()))",
"sys.stdout.flush()")
def checkOutput(output):
self.assertEquals('[0, 1, 2, 3]', output)
reactor = self.buildReactor()
class Protocol(ProcessProtocol):
def __init__(self):
self.output = []
def outReceived(self, data):
self.output.append(data)
def processEnded(self, reason):
try:
checkOutput("".join(self.output))
finally:
reactor.stop()
proto = Protocol()
reactor.callWhenRunning(
reactor.spawnProcess, proto, sys.executable,
[sys.executable, "-Wignore", "-c", "\n".join(source)],
usePTY=self.usePTY)
self.runReactor(reactor)
def postOptions(self):
if self["journald"]:
destination = JournaldDestination()
else:
if self["logfile"] is None:
logfile = self._sys_module.stdout
else:
logfilepath = FilePath(self["logfile"])
logfilepath_directory = logfilepath.parent()
if not logfilepath_directory.exists():
logfilepath_directory.makedirs()
# A twisted.python.logfile which has write and flush methods
# but which also rotates the log file.
logfile = LogFile.fromFullPath(
logfilepath.path, rotateLength=LOGFILE_LENGTH, maxRotatedFiles=LOGFILE_COUNT
)
destination = FileDestination(file=logfile)
self.eliot_destination = destination
original_postOptions(self)
def test_cssCached(self):
"""
The processed CSS file is cached, and updated when the underlying
file changes.
"""
clock = Clock()
fc = FileCache(lambda: clock.seconds(), 1)
temp = FilePath(self.mktemp() + '.css')
temp.setContent("p { color: red; }\n")
bf = BetterFile(temp.parent().path, fileCache=fc, rewriteCss=True)
d = self._requestPostpathAndRender(bf, [temp.basename()])
def assertColorRed((request, child)):
lines = "".join(request.written).split("\n")
self.assertEqual(["p { color: red; }", ""], lines[1:])
d.addCallback(assertColorRed)
def modifyUnderlyingAndMakeRequest(_):
with temp.open('wb') as f:
f.write("p { color: green; }\n")
d = self._requestPostpathAndRender(bf, [temp.basename()])
return d
d.addCallback(modifyUnderlyingAndMakeRequest)
def assertStillColorRed((request, child)):
lines = "".join(request.written).split("\n")
self.assertEqual(["p { color: red; }", ""], lines[1:])
d.addCallback(assertStillColorRed)
def advanceClockAndMakeRequest(_):
clock.advance(1)
d = self._requestPostpathAndRender(bf, [temp.basename()])
return d
d.addCallback(advanceClockAndMakeRequest)
def assertColorGreen((request, child)):
lines = "".join(request.written).split("\n")
self.assertEqual(["p { color: green; }", ""], lines[1:])
d.addCallback(assertColorGreen)
return d
def _getAxiomaticScript(self):
here = FilePath(__file__)
# Try to find it relative to the source of this test.
bin = here.parent().parent().parent().child("bin")
axiomatic = bin.child("axiomatic")
if axiomatic.exists():
# Great, use that one.
axiomatic = axiomatic.path
else:
# Try to find it on the path, instead.
axiomatics = which("axiomatic")
if axiomatics:
# Great, it was on the path.
axiomatic = axiomatics[0]
else:
# Nope, not there, give up.
raise SkipTest(
"Could not find axiomatic script on path or at %s" % (
axiomatic.path,))
return axiomatic
def _getAxiomaticScript(self):
here = FilePath(__file__)
# Try to find it relative to the source of this test.
bin = here.parent().parent().parent().child("bin")
axiomatic = bin.child("axiomatic")
if axiomatic.exists():
# Great, use that one.
axiomatic = axiomatic.path
else:
# Try to find it on the path, instead.
axiomatics = which("axiomatic")
if axiomatics:
# Great, it was on the path.
axiomatic = axiomatics[0]
else:
# Nope, not there, give up.
raise SkipTest(
"Could not find axiomatic script on path or at %s" %
(axiomatic.path, ))
return axiomatic
def postOptions(self):
if self['journald']:
destination = JournaldDestination()
else:
if self['logfile'] is None:
logfile = self._sys_module.stdout
else:
logfilepath = FilePath(self['logfile'])
logfilepath_directory = logfilepath.parent()
if not logfilepath_directory.exists():
logfilepath_directory.makedirs()
# A twisted.python.logfile which has write and flush methods
# but which also rotates the log file.
logfile = LogFile.fromFullPath(
logfilepath.path,
rotateLength=LOGFILE_LENGTH,
maxRotatedFiles=LOGFILE_COUNT,
)
destination = FileDestination(file=logfile)
self.eliot_destination = destination
original_postOptions(self)
def namesToPaths(projectNames, verbose=True):
"""
Turn names of projects into project objects.
@type projectNames: C{list} of C{str}
@param projectNames: The names of Python packages in each project to find
and for which to create a L{Project} instance.
@type verbose: C{bool}
@param verbose: If true, report some information about what happens to
stdout.
@rtype: C{list} of L{Project}
@return: Project instances for each package in C{projectNames}, with
C{name}, C{initPath}, and C{package} set.
"""
projectObjects = []
for projName in projectNames:
for pfx in '', 'x':
try:
projPackage = namedModule(pfx + projName)
except ImportError:
pass
else:
projPackagePath = FilePath(projPackage.__file__)
break
else:
raise SystemExit("Failed to find Python package for %s." %
(projName, ))
realName = projPackagePath.parent().parent().basename()
projectObjects.append(
Project(name=realName,
initPath=projPackagePath,
package=projPackage,
version=None))
if verbose:
print 'Found', projName, 'as', realName, 'at', projPackagePath.path
return projectObjects
def namesToPaths(projectNames, verbose=True):
"""
Turn names of projects into project objects.
@type projectNames: C{list} of C{str}
@param projectNames: The names of Python packages in each project to find
and for which to create a L{Project} instance.
@type verbose: C{bool}
@param verbose: If true, report some information about what happens to
stdout.
@rtype: C{list} of L{Project}
@return: Project instances for each package in C{projectNames}, with
C{name}, C{initPath}, and C{package} set.
"""
projectObjects = []
for projName in projectNames:
for pfx in '', 'x':
try:
projPackage = namedModule(pfx + projName)
except ImportError:
pass
else:
projPackagePath = FilePath(projPackage.__file__)
break
else:
raise SystemExit("Failed to find Python package for %s."
% (projName,))
realName = projPackagePath.parent().parent().basename()
projectObjects.append(Project(name=realName,
initPath=projPackagePath,
package=projPackage,
version=None))
if verbose:
print 'Found', projName, 'as', realName, 'at', projPackagePath.path
return projectObjects
def test_toptobottomMissingSource(self):
"""
--order=toptobottom detects the source line of methods from modules
whose source file is missing.
"""
tempdir = self.mktemp()
package = FilePath(tempdir).child('twisted_toptobottom_temp')
package.makedirs()
package.child('__init__.py').setContent(b'')
package.child('test_missing.py').setContent(textwrap.dedent('''
from twisted.trial.unittest import TestCase
class TestMissing(TestCase):
def test_second(self): pass
def test_third(self): pass
def test_fourth(self): pass
def test_first(self): pass
''').encode('utf8'))
pathEntry = package.parent().path
sys.path.insert(0, pathEntry)
self.addCleanup(sys.path.remove, pathEntry)
from twisted_toptobottom_temp import test_missing
self.addCleanup(sys.modules.pop, 'twisted_toptobottom_temp')
self.addCleanup(sys.modules.pop, test_missing.__name__)
package.child('test_missing.py').remove()
self.config.parseOptions([
"--order", "toptobottom", "twisted.trial.test.ordertests"])
loader = trial._getLoader(self.config)
suite = loader.loadModule(test_missing)
self.assertEqual(
testNames(suite), [
'twisted_toptobottom_temp.test_missing.TestMissing.test_second',
'twisted_toptobottom_temp.test_missing.TestMissing.test_third',
'twisted_toptobottom_temp.test_missing.TestMissing.test_fourth',
'twisted_toptobottom_temp.test_missing.TestMissing.test_first'])
class GetExtensionsTest(TestCase):
"""
Tests for L{dist.getExtensions}.
"""
setupTemplate = (
"from twisted.python.dist import ConditionalExtension\n"
"extensions = [\n"
" ConditionalExtension(\n"
" '%s', ['twisted/some/thing.c'],\n"
" condition=lambda builder: True)\n"
" ]\n")
def setUp(self):
self.basedir = FilePath(self.mktemp()).child("twisted")
self.basedir.makedirs()
self.addCleanup(os.chdir, os.getcwd())
os.chdir(self.basedir.parent().path)
def writeSetup(self, name, *path):
"""
Write out a C{setup.py} file to a location determined by
L{self.basedir} and L{path}. L{self.setupTemplate} is used to
generate its contents.
"""
outdir = self.basedir.descendant(path)
outdir.makedirs()
setup = outdir.child("setup.py")
setup.setContent(self.setupTemplate % (name,))
def writeEmptySetup(self, *path):
"""
Write out an empty C{setup.py} file to a location determined by
L{self.basedir} and L{path}.
"""
outdir = self.basedir.descendant(path)
outdir.makedirs()
outdir.child("setup.py").setContent("")
def assertExtensions(self, expected):
"""
Assert that the given names match the (sorted) names of discovered
extensions.
"""
extensions = dist.getExtensions()
names = [extension.name for extension in extensions]
self.assertEqual(sorted(names), expected)
def test_getExtensions(self):
"""
Files named I{setup.py} in I{twisted/topfiles} and I{twisted/*/topfiles}
are executed with L{execfile} in order to discover the extensions they
declare.
"""
self.writeSetup("twisted.transmutate", "topfiles")
self.writeSetup("twisted.tele.port", "tele", "topfiles")
self.assertExtensions(["twisted.tele.port", "twisted.transmutate"])
def test_getExtensionsTooDeep(self):
"""
Files named I{setup.py} in I{topfiles} directories are not considered if
they are too deep in the directory hierarchy.
"""
self.writeSetup("twisted.trans.mog.rify", "trans", "mog", "topfiles")
self.assertExtensions([])
def test_getExtensionsNotTopfiles(self):
"""
The folder in which I{setup.py} is discovered must be called I{topfiles}
otherwise it is ignored.
"""
self.writeSetup("twisted.metamorphosis", "notfiles")
self.assertExtensions([])
def test_getExtensionsNotSupportedOnJava(self):
"""
Extensions are not supported on Java-based platforms.
"""
self.addCleanup(setattr, sys, "platform", sys.platform)
sys.platform = "java"
self.writeSetup("twisted.sorcery", "topfiles")
self.assertExtensions([])
def test_getExtensionsExtensionsLocalIsOptional(self):
"""
It is acceptable for extensions to not define the C{extensions} local
variable.
"""
self.writeEmptySetup("twisted.necromancy", "topfiles")
self.assertExtensions([])
import json
from collections import defaultdict
r_title = re.compile('<title>(.*?)</title>', re.I | re.M | re.S)
def getInfo(fp):
global r_title
segments = fp.path.split('/')
section = segments[-2]
name = title = segments[-1]
guts = fp.getContent()
m = r_title.search(guts)
if m:
title = m.groups()[0]
return section, name, title
def main(starting_path):
r = defaultdict(lambda: [])
for f in starting_path.walk():
if f.isfile():
section, name, title = getInfo(f)
r[section].append({'name': name, 'title': title})
return r
if __name__ == '__main__':
me = FilePath(__file__)
d = main(me.parent().parent().child('articles'))
print json.dumps(d)
class WarnAboutFunctionTests(SynchronousTestCase):
"""
Tests for L{twisted.python.deprecate.warnAboutFunction} which allows the
callers of a function to issue a C{DeprecationWarning} about that function.
"""
def setUp(self):
"""
Create a file that will have known line numbers when emitting warnings.
"""
self.package = FilePath(
self.mktemp().encode("utf-8")).child(b'twisted_private_helper')
self.package.makedirs()
self.package.child(b'__init__.py').setContent(b'')
self.package.child(b'module.py').setContent(b'''
"A module string"
from twisted.python import deprecate
def testFunction():
"A doc string"
a = 1 + 2
return a
def callTestFunction():
b = testFunction()
if b == 3:
deprecate.warnAboutFunction(testFunction, "A Warning String")
''')
# Python 3 doesn't accept bytes in sys.path:
packagePath = self.package.parent().path.decode("utf-8")
sys.path.insert(0, packagePath)
self.addCleanup(sys.path.remove, packagePath)
modules = sys.modules.copy()
self.addCleanup(lambda:
(sys.modules.clear(), sys.modules.update(modules)))
def test_warning(self):
"""
L{deprecate.warnAboutFunction} emits a warning the file and line number
of which point to the beginning of the implementation of the function
passed to it.
"""
def aFunc():
pass
deprecate.warnAboutFunction(aFunc, 'A Warning Message')
warningsShown = self.flushWarnings()
filename = __file__
if filename.lower().endswith('.pyc'):
filename = filename[:-1]
self.assertSamePath(FilePath(warningsShown[0]["filename"]),
FilePath(filename))
self.assertEqual(warningsShown[0]["message"], "A Warning Message")
def test_warningLineNumber(self):
"""
L{deprecate.warnAboutFunction} emits a C{DeprecationWarning} with the
number of a line within the implementation of the function passed to it.
"""
from twisted_private_helper import module
module.callTestFunction()
warningsShown = self.flushWarnings()
self.assertSamePath(
FilePath(warningsShown[0]["filename"].encode("utf-8")),
self.package.sibling(b'twisted_private_helper').child(
b'module.py'))
# Line number 9 is the last line in the testFunction in the helper
# module.
self.assertEqual(warningsShown[0]["lineno"], 9)
self.assertEqual(warningsShown[0]["message"], "A Warning String")
self.assertEqual(len(warningsShown), 1)
def assertSamePath(self, first, second):
"""
Assert that the two paths are the same, considering case normalization
appropriate for the current platform.
@type first: L{FilePath}
@type second: L{FilePath}
@raise C{self.failureType}: If the paths are not the same.
"""
self.assertTrue(
normcase(first.path) == normcase(second.path),
"%r != %r" % (first, second))
def test_renamedFile(self):
"""
Even if the implementation of a deprecated function is moved around on
the filesystem, the line number in the warning emitted by
L{deprecate.warnAboutFunction} points to a line in the implementation of
the deprecated function.
"""
from twisted_private_helper import module
# Clean up the state resulting from that import; we're not going to use
# this module, so it should go away.
del sys.modules['twisted_private_helper']
del sys.modules[module.__name__]
# Rename the source directory
self.package.moveTo(self.package.sibling(b'twisted_renamed_helper'))
# Make sure importlib notices we've changed importable packages:
if invalidate_caches:
invalidate_caches()
# Import the newly renamed version
from twisted_renamed_helper import module
self.addCleanup(sys.modules.pop, 'twisted_renamed_helper')
self.addCleanup(sys.modules.pop, module.__name__)
module.callTestFunction()
warningsShown = self.flushWarnings()
warnedPath = FilePath(warningsShown[0]["filename"].encode("utf-8"))
expectedPath = self.package.sibling(b'twisted_renamed_helper').child(
b'module.py')
self.assertSamePath(warnedPath, expectedPath)
self.assertEqual(warningsShown[0]["lineno"], 9)
self.assertEqual(warningsShown[0]["message"], "A Warning String")
self.assertEqual(len(warningsShown), 1)
def test_filteredWarning(self):
"""
L{deprecate.warnAboutFunction} emits a warning that will be filtered if
L{warnings.filterwarning} is called with the module name of the
deprecated function.
"""
# Clean up anything *else* that might spuriously filter out the warning,
# such as the "always" simplefilter set up by unittest._collectWarnings.
# We'll also rely on trial to restore the original filters afterwards.
del warnings.filters[:]
warnings.filterwarnings(action="ignore",
module="twisted_private_helper")
from twisted_private_helper import module
module.callTestFunction()
warningsShown = self.flushWarnings()
self.assertEqual(len(warningsShown), 0)
def test_filteredOnceWarning(self):
"""
L{deprecate.warnAboutFunction} emits a warning that will be filtered
once if L{warnings.filterwarning} is called with the module name of the
deprecated function and an action of once.
"""
# Clean up anything *else* that might spuriously filter out the warning,
# such as the "always" simplefilter set up by unittest._collectWarnings.
# We'll also rely on trial to restore the original filters afterwards.
del warnings.filters[:]
warnings.filterwarnings(action="module",
module="twisted_private_helper")
from twisted_private_helper import module
module.callTestFunction()
module.callTestFunction()
warningsShown = self.flushWarnings()
self.assertEqual(len(warningsShown), 1)
message = warningsShown[0]['message']
category = warningsShown[0]['category']
filename = warningsShown[0]['filename']
lineno = warningsShown[0]['lineno']
msg = warnings.formatwarning(message, category, filename, lineno)
self.assertTrue(
msg.endswith("module.py:9: DeprecationWarning: A Warning String\n"
" return a\n"),
"Unexpected warning string: %r" % (msg, ))
class GetExtensionsTests(TestCase):
"""
Tests for L{dist.getExtensions}.
"""
setupTemplate = ("from twisted.python.dist import ConditionalExtension\n"
"extensions = [\n"
" ConditionalExtension(\n"
" '%s', ['twisted/some/thing.c'],\n"
" condition=lambda builder: True)\n"
" ]\n")
def setUp(self):
self.basedir = FilePath(self.mktemp()).child("twisted")
self.basedir.makedirs()
self.addCleanup(os.chdir, os.getcwd())
os.chdir(self.basedir.parent().path)
def writeSetup(self, name, *path):
"""
Write out a C{setup.py} file to a location determined by
L{self.basedir} and L{path}. L{self.setupTemplate} is used to
generate its contents.
"""
outdir = self.basedir.descendant(path)
outdir.makedirs()
setup = outdir.child("setup.py")
setup.setContent(self.setupTemplate % (name, ))
def writeEmptySetup(self, *path):
"""
Write out an empty C{setup.py} file to a location determined by
L{self.basedir} and L{path}.
"""
outdir = self.basedir.descendant(path)
outdir.makedirs()
outdir.child("setup.py").setContent("")
def assertExtensions(self, expected):
"""
Assert that the given names match the (sorted) names of discovered
extensions.
"""
extensions = dist.getExtensions()
names = [extension.name for extension in extensions]
self.assertEqual(sorted(names), expected)
def test_getExtensions(self):
"""
Files named I{setup.py} in I{twisted/topfiles} and I{twisted/*/topfiles}
are executed with L{execfile} in order to discover the extensions they
declare.
"""
self.writeSetup("twisted.transmutate", "topfiles")
self.writeSetup("twisted.tele.port", "tele", "topfiles")
self.assertExtensions(["twisted.tele.port", "twisted.transmutate"])
def test_getExtensionsTooDeep(self):
"""
Files named I{setup.py} in I{topfiles} directories are not considered if
they are too deep in the directory hierarchy.
"""
self.writeSetup("twisted.trans.mog.rify", "trans", "mog", "topfiles")
self.assertExtensions([])
def test_getExtensionsNotTopfiles(self):
"""
The folder in which I{setup.py} is discovered must be called I{topfiles}
otherwise it is ignored.
"""
self.writeSetup("twisted.metamorphosis", "notfiles")
self.assertExtensions([])
def test_getExtensionsNotSupportedOnJava(self):
"""
Extensions are not supported on Java-based platforms.
"""
self.addCleanup(setattr, sys, "platform", sys.platform)
sys.platform = "java"
self.writeSetup("twisted.sorcery", "topfiles")
self.assertExtensions([])
def test_getExtensionsExtensionsLocalIsOptional(self):
"""
It is acceptable for extensions to not define the C{extensions} local
variable.
"""
self.writeEmptySetup("twisted.necromancy", "topfiles")
self.assertExtensions([])
def test_openFileDescriptors(self):
"""
Processes spawned with spawnProcess() close all extraneous file
descriptors in the parent. They do have a stdin, stdout, and stderr
open.
"""
# To test this, we are going to open a file descriptor in the parent
# that is unlikely to be opened in the child, then verify that it's not
# open in the child.
here = FilePath(__file__)
top = here.parent().parent().parent().parent()
source = ("import sys", "sys.path.insert(0, '%s')" % (top.path, ),
"from twisted.internet import process",
"sys.stdout.write(repr(process._listOpenFDs()))",
"sys.stdout.flush()")
r, w = os.pipe()
self.addCleanup(os.close, r)
self.addCleanup(os.close, w)
# The call to "os.listdir()" (in _listOpenFDs's implementation) opens a
# file descriptor (with "opendir"), which shows up in _listOpenFDs's
# result. And speaking of "random" file descriptors, the code required
# for _listOpenFDs itself imports logger, which imports random, which
# (depending on your Python version) might leave /dev/urandom open.
# More generally though, even if we were to use an extremely minimal C
# program, the operating system would be within its rights to open file
# descriptors we might not know about in the C library's
# initialization; things like debuggers, profilers, or nsswitch plugins
# might open some and this test should pass in those environments.
# Although some of these file descriptors aren't predictable, we should
# at least be able to select a very large file descriptor which is very
# unlikely to be opened automatically in the subprocess. (Apply a
# fudge factor to avoid hard-coding something too near a limit
# condition like the maximum possible file descriptor, which a library
# might at least hypothetically select.)
fudgeFactor = 17
unlikelyFD = (resource.getrlimit(resource.RLIMIT_NOFILE)[0] -
fudgeFactor)
os.dup2(w, unlikelyFD)
self.addCleanup(os.close, unlikelyFD)
output = io.BytesIO()
class GatheringProtocol(ProcessProtocol):
outReceived = output.write
def processEnded(self, reason):
reactor.stop()
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess,
GatheringProtocol(),
sys.executable,
[sys.executable, "-Wignore", "-c", "\n".join(source)],
env=os.environ,
usePTY=self.usePTY)
self.runReactor(reactor)
reportedChildFDs = set(eval(output.getvalue()))
stdFDs = [0, 1, 2]
# Unfortunately this assertion is still not *entirely* deterministic,
# since hypothetically, any library could open any file descriptor at
# any time. See comment above.
self.assertEqual(
reportedChildFDs.intersection(set(stdFDs + [unlikelyFD])),
set(stdFDs))
from twisted.python.filepath import FilePath
from jinja2 import Environment, FileSystemLoader
import json
pkg_root = FilePath(__file__).parent()
doc_root = pkg_root.parent().child('docs')
env = Environment(loader=FileSystemLoader(pkg_root.child('templates').path))
def render(name, params):
template = env.get_template(name)
return template.render(params)
def renderTo(dst, name, params):
dst.setContent(render(name, params))
def renderResourceSchema(name, schema):
schema_root = doc_root.child('resources')
doctypes = ['identity', 'observation', 'prescription']
for doctype in doctypes:
doctype_schema = schema.get(doctype, None)
if not doctype_schema:
continue
renderTo(schema_root.child('%s.%s.rst' % (name,doctype)), 'schema.rst', {
'schema': doctype_schema,
})
def test_openFileDescriptors(self):
"""
Processes spawned with spawnProcess() close all extraneous file
descriptors in the parent. They do have a stdin, stdout, and stderr
open.
"""
# To test this, we are going to open a file descriptor in the parent
# that is unlikely to be opened in the child, then verify that it's not
# open in the child.
here = FilePath(__file__)
top = here.parent().parent().parent().parent()
source = (
"import sys",
"sys.path.insert(0, '%s')" % (top.path,),
"from twisted.internet import process",
"sys.stdout.write(repr(process._listOpenFDs()))",
"sys.stdout.flush()")
r, w = os.pipe()
self.addCleanup(os.close, r)
self.addCleanup(os.close, w)
# The call to "os.listdir()" (in _listOpenFDs's implementation) opens a
# file descriptor (with "opendir"), which shows up in _listOpenFDs's
# result. And speaking of "random" file descriptors, the code required
# for _listOpenFDs itself imports logger, which imports random, which
# (depending on your Python version) might leave /dev/urandom open.
# More generally though, even if we were to use an extremely minimal C
# program, the operating system would be within its rights to open file
# descriptors we might not know about in the C library's
# initialization; things like debuggers, profilers, or nsswitch plugins
# might open some and this test should pass in those environments.
# Although some of these file descriptors aren't predictable, we should
# at least be able to select a very large file descriptor which is very
# unlikely to be opened automatically in the subprocess. (Apply a
# fudge factor to avoid hard-coding something too near a limit
# condition like the maximum possible file descriptor, which a library
# might at least hypothetically select.)
fudgeFactor = 17
unlikelyFD = (resource.getrlimit(resource.RLIMIT_NOFILE)[0]
- fudgeFactor)
os.dup2(w, unlikelyFD)
self.addCleanup(os.close, unlikelyFD)
output = io.BytesIO()
class GatheringProtocol(ProcessProtocol):
outReceived = output.write
def processEnded(self, reason):
reactor.stop()
reactor = self.buildReactor()
reactor.callWhenRunning(
reactor.spawnProcess, GatheringProtocol(), sys.executable,
[sys.executable, "-Wignore", "-c", "\n".join(source)],
env=os.environ, usePTY=self.usePTY)
self.runReactor(reactor)
reportedChildFDs = set(eval(output.getvalue()))
stdFDs = [0, 1, 2]
# Unfortunately this assertion is still not *entirely* deterministic,
# since hypothetically, any library could open any file descriptor at
# any time. See comment above.
self.assertEqual(
reportedChildFDs.intersection(set(stdFDs + [unlikelyFD])),
set(stdFDs)
)
def test_template_root(self):
"""
Should be in mold/templates
"""
me = FilePath(__file__)
self.assertEqual(template_root, me.parent().parent().child('templates'))
def test_reactorSelection(self):
"""
L{AxiomaticStart} optionally takes the name of a reactor and
installs it instead of the default reactor.
"""
# Since this process is already hopelessly distant from the state in
# which I{axiomatic start} operates, it would make no sense to try a
# functional test of this behavior in this process. Since the
# behavior being tested involves lots of subtle interactions between
# lots of different pieces of code (the reactor might get installed
# at the end of a ten-deep chain of imports going through as many
# different projects), it also makes no sense to try to make this a
# unit test. So, start a child process and try to use the alternate
# reactor functionality there.
here = FilePath(__file__)
# Try to find it relative to the source of this test.
bin = here.parent().parent().parent().child("bin")
axiomatic = bin.child("axiomatic")
if axiomatic.exists():
# Great, use that one.
axiomatic = axiomatic.path
else:
# Try to find it on the path, instead.
axiomatics = which("axiomatic")
if axiomatics:
# Great, it was on the path.
axiomatic = axiomatics[0]
else:
# Nope, not there, give up.
raise SkipTest(
"Could not find axiomatic script on path or at %s" %
(axiomatic.path, ))
# Create a store for the child process to use and put an item in it.
# This will force an import of the module that defines that item's
# class when the child process starts. The module imports the default
# reactor at the top-level, making this the worst-case for the reactor
# selection code.
storePath = self.mktemp()
store = Store(storePath)
SomeItem(store=store)
store.close()
# Install select reactor because it available on all platforms, and
# it is still an error to try to install the select reactor even if
# the already installed reactor was the select reactor.
argv = [
sys.executable, axiomatic, "-d", storePath, "start", "--reactor",
"select", "-n"
]
expected = [
"reactor class: twisted.internet.selectreactor.SelectReactor.",
"reactor class: <class 'twisted.internet.selectreactor.SelectReactor'>"
]
proto, complete = AxiomaticStartProcessProtocol.protocolAndDeferred(
expected)
# Make sure the version of Axiom under test is found by the child
# process.
import axiom, epsilon
environ = os.environ.copy()
environ['PYTHONPATH'] = os.pathsep.join([
FilePath(epsilon.__file__).parent().parent().path,
FilePath(axiom.__file__).parent().parent().path,
environ['PYTHONPATH']
])
reactor.spawnProcess(proto, sys.executable, argv, env=environ)
return complete
def assertEqualsLastRecording(self, basename):
jsonPath = here.child('data').child(basename + '.json')
recorded = Instrument.recorder.toDict()
recorded = json.loads(json.dumps(recorded))
if not jsonPath.exists():
print 'Serializing recording to %s' % jsonPath.path
fd = jsonPath.open('w')
fd.write(json.dumps(recorded))
fd.close()
d = json.loads(jsonPath.open('r').read())
self.assertEqual(recorded, d)
def method(basename):
def test_song(self):
mod = namedAny('tutor.%s' % basename)
d = self.spin(96 * 8)
def check(result):
self.assertEqualsLastRecording(basename)
return d.addCallback(check)
return test_song
for child in here.parent().globChildren('song*'):
basename, _ = os.path.splitext(child.basename())
setattr(SongsAreOkTestCase, 'test_%s' % basename, method(basename))
class WarnAboutFunctionTests(SynchronousTestCase):
"""
Tests for L{twisted.python.deprecate.warnAboutFunction} which allows the
callers of a function to issue a C{DeprecationWarning} about that function.
"""
def setUp(self):
"""
Create a file that will have known line numbers when emitting warnings.
"""
self.package = FilePath(self.mktemp().encode("utf-8")
).child(b'twisted_private_helper')
self.package.makedirs()
self.package.child(b'__init__.py').setContent(b'')
self.package.child(b'module.py').setContent(b'''
"A module string"
from twisted.python import deprecate
def testFunction():
"A doc string"
a = 1 + 2
return a
def callTestFunction():
b = testFunction()
if b == 3:
deprecate.warnAboutFunction(testFunction, "A Warning String")
''')
# Python 3 doesn't accept bytes in sys.path:
packagePath = self.package.parent().path.decode("utf-8")
sys.path.insert(0, packagePath)
self.addCleanup(sys.path.remove, packagePath)
modules = sys.modules.copy()
self.addCleanup(
lambda: (sys.modules.clear(), sys.modules.update(modules)))
# On Windows on Python 3, most FilePath interactions produce
# DeprecationWarnings, so flush them here so that they don't interfere
# with the tests.
if platform.isWindows() and _PY3:
self.flushWarnings()
def test_warning(self):
"""
L{deprecate.warnAboutFunction} emits a warning the file and line number
of which point to the beginning of the implementation of the function
passed to it.
"""
def aFunc():
pass
deprecate.warnAboutFunction(aFunc, 'A Warning Message')
warningsShown = self.flushWarnings()
filename = __file__
if filename.lower().endswith('.pyc'):
filename = filename[:-1]
self.assertSamePath(
FilePath(warningsShown[0]["filename"]), FilePath(filename))
self.assertEqual(warningsShown[0]["message"], "A Warning Message")
def test_warningLineNumber(self):
"""
L{deprecate.warnAboutFunction} emits a C{DeprecationWarning} with the
number of a line within the implementation of the function passed to it.
"""
from twisted_private_helper import module
module.callTestFunction()
warningsShown = self.flushWarnings()
self.assertSamePath(
FilePath(warningsShown[0]["filename"].encode("utf-8")),
self.package.sibling(b'twisted_private_helper').child(b'module.py'))
# Line number 9 is the last line in the testFunction in the helper
# module.
self.assertEqual(warningsShown[0]["lineno"], 9)
self.assertEqual(warningsShown[0]["message"], "A Warning String")
self.assertEqual(len(warningsShown), 1)
def assertSamePath(self, first, second):
"""
Assert that the two paths are the same, considering case normalization
appropriate for the current platform.
@type first: L{FilePath}
@type second: L{FilePath}
@raise C{self.failureType}: If the paths are not the same.
"""
self.assertTrue(
normcase(first.path) == normcase(second.path),
"%r != %r" % (first, second))
def test_renamedFile(self):
"""
Even if the implementation of a deprecated function is moved around on
the filesystem, the line number in the warning emitted by
L{deprecate.warnAboutFunction} points to a line in the implementation of
the deprecated function.
"""
from twisted_private_helper import module
# Clean up the state resulting from that import; we're not going to use
# this module, so it should go away.
del sys.modules['twisted_private_helper']
del sys.modules[module.__name__]
# Rename the source directory
self.package.moveTo(self.package.sibling(b'twisted_renamed_helper'))
# Make sure importlib notices we've changed importable packages:
if invalidate_caches:
invalidate_caches()
# Import the newly renamed version
from twisted_renamed_helper import module
self.addCleanup(sys.modules.pop, 'twisted_renamed_helper')
self.addCleanup(sys.modules.pop, module.__name__)
module.callTestFunction()
warningsShown = self.flushWarnings([module.testFunction])
warnedPath = FilePath(warningsShown[0]["filename"].encode("utf-8"))
expectedPath = self.package.sibling(
b'twisted_renamed_helper').child(b'module.py')
self.assertSamePath(warnedPath, expectedPath)
self.assertEqual(warningsShown[0]["lineno"], 9)
self.assertEqual(warningsShown[0]["message"], "A Warning String")
self.assertEqual(len(warningsShown), 1)
def test_filteredWarning(self):
"""
L{deprecate.warnAboutFunction} emits a warning that will be filtered if
L{warnings.filterwarning} is called with the module name of the
deprecated function.
"""
# Clean up anything *else* that might spuriously filter out the warning,
# such as the "always" simplefilter set up by unittest._collectWarnings.
# We'll also rely on trial to restore the original filters afterwards.
del warnings.filters[:]
warnings.filterwarnings(
action="ignore", module="twisted_private_helper")
from twisted_private_helper import module
module.callTestFunction()
warningsShown = self.flushWarnings()
self.assertEqual(len(warningsShown), 0)
def test_filteredOnceWarning(self):
"""
L{deprecate.warnAboutFunction} emits a warning that will be filtered
once if L{warnings.filterwarning} is called with the module name of the
deprecated function and an action of once.
"""
# Clean up anything *else* that might spuriously filter out the warning,
# such as the "always" simplefilter set up by unittest._collectWarnings.
# We'll also rely on trial to restore the original filters afterwards.
del warnings.filters[:]
warnings.filterwarnings(
action="module", module="twisted_private_helper")
from twisted_private_helper import module
module.callTestFunction()
module.callTestFunction()
warningsShown = self.flushWarnings()
self.assertEqual(len(warningsShown), 1)
message = warningsShown[0]['message']
category = warningsShown[0]['category']
filename = warningsShown[0]['filename']
lineno = warningsShown[0]['lineno']
msg = warnings.formatwarning(message, category, filename, lineno)
self.assertTrue(
msg.endswith("module.py:9: DeprecationWarning: A Warning String\n"
" return a\n"),
"Unexpected warning string: %r" % (msg,))
yield d
def assertEqualsLastRecording(self, basename):
jsonPath = here.child('data').child(basename + '.json')
recorded = Instrument.recorder.toDict()
recorded = json.loads(json.dumps(recorded))
if not jsonPath.exists():
print 'Serializing recording to %s' % jsonPath.path
fd = jsonPath.open('w')
fd.write(json.dumps(recorded))
fd.close()
d = json.loads(jsonPath.open('r').read())
self.assertEqual(recorded, d)
def method(basename):
def test_song(self):
mod = namedAny('tutor.%s' % basename)
d = self.spin(96 * 8)
def check(result):
self.assertEqualsLastRecording(basename)
return d.addCallback(check)
return test_song
for child in here.parent().globChildren('song*'):
basename, _ = os.path.splitext(child.basename())
setattr(SongsAreOkTestCase, 'test_%s' % basename, method(basename))
