def test_upgrade_configuration_versions(self, versions):
"""
A range of versions can be upgraded and the configuration
blob after upgrade will match that which is expected for the
particular version.
See flocker/control/test/configurations for individual
version JSON files and generation code.
"""
source_version, target_version = versions
configs_dir = FilePath(__file__).sibling('configurations')
# Choose the latest configuration number available for the given
# version of the config. The configuration has increased in complexity
# over time, so we have added additional configurations to verify that
# the new fields can be correctly upgraded.
source_json_glob = b"configuration_*_v%d.json" % source_version
source_jsons = sorted(configs_dir.globChildren(source_json_glob),
key=lambda x: x.basename())
config_num = int(source_jsons[-1].basename().split('_')[1])
source_json_file = b"configuration_%d_v%d.json" % (config_num,
versions[0])
target_json_file = b"configuration_%d_v%d.json" % (config_num,
versions[1])
source_json = configs_dir.child(source_json_file).getContent()
target_json = configs_dir.child(target_json_file).getContent()
upgraded_json = migrate_configuration(
source_version, target_version,
source_json, ConfigurationMigration)
self.assertEqual(json.loads(upgraded_json), json.loads(target_json))
def test_switch(self):
"""
Switch should work
"""
runner = Runner()
# I'm getting AF_UNIX path too long errors using self.mktemp()
base = FilePath(tempfile.mkdtemp())
log.msg('tmpdir: %r' % base.path)
root = base.child('root')
src = base.child('src')
dst = base.child('dst')
_ = yield runner.start(root.path, 'unix:'+src.path, 'unix:'+dst.path)
pidfile = root.child('grace.pid')
pid = pidfile.getContent()
self.addCleanup(self.kill, pid)
r = yield runner.switch(root.path, 'unix:'+src.path, 'unix:/foo')
r = yield runner.ls(root.path)
self.assertEqual(r, [
{
'src': 'unix:'+src.path,
'dst': 'unix:/foo',
'conns': 0,
'active': True,
}
], "Should have switched")
def test_start_waits(self):
"""
Start should not return until it's actually working.
"""
runner = Runner()
base = FilePath(tempfile.mkdtemp())
root = base.child('root')
src = base.child('src')
dst = base.child('dst')
_ = yield runner.start(root.path, 'unix:'+src.path, 'unix:'+dst.path)
self.assertTrue(root.child('grace.pid').exists(), "Should have a pid")
pid = root.child('grace.pid').getContent().strip()
self.addCleanup(self.kill, pid)
self.assertTrue(root.child('grace.socket').exists(), "Should have a "
"socket")
self.assertTrue(root.exists(), "Should have made the root dir")
tac = root.child('grace.tac')
self.assertTrue(tac.exists(), "Should have made grace.tac")
self.assertEqual(tac.getContent(),
getTac(('unix:'+src.path, 'unix:'+dst.path)),
"Should have made the tac file using getTac")
def test_load_error_on_unreadable_key_file(self):
"""
A ``PathError`` is raised if the key file path given to
``UserCredential.from_path`` cannot be opened for reading.
"""
path = FilePath(self.mktemp())
path.makedirs()
crt_path = path.child(self.cert_file_name)
crt_file = crt_path.open(b'w')
crt_file.write(b"dummy")
crt_file.close()
key_path = path.child(self.key_file_name)
key_file = key_path.open(b'w')
key_file.write(b"dummy")
key_file.close()
# make file unreadable
key_path.chmod(0o100)
e = self.assertRaises(
PathError, cls.from_path,
path, **kwargs
)
expected = (
"Private key file could not be opened. "
"Permission denied {path}"
).format(path=key_path.path)
self.assertEqual(str(e), expected)
def test_stop(self):
"""
Stop will stop a running process.
"""
runner = Runner()
# I'm getting AF_UNIX path too long errors using self.mktemp()
base = FilePath(tempfile.mkdtemp())
log.msg('tmpdir: %r' % base.path)
root = base.child('root')
src = base.child('src')
dst = base.child('dst')
_ = yield runner.start(root.path, 'unix:'+src.path, 'unix:'+dst.path)
pidfile = root.child('grace.pid')
pid = pidfile.getContent()
self.addCleanup(self.kill, pid)
_ = yield runner.stop(root.path)
# tail the log until you see Server Shut Down
# XXX stop should maybe do the same... so that it doesn't return until
# the process has actually stopped.
logfile = root.child('grace.log')
self.assertTrue(logfile.exists())
_ = yield self.tailUntil(logfile.path, 'Server Shut Down.')
self.assertFalse(pidfile.exists(), "pidfile should be gone: %r" % pidfile.path)
def test_addPyListings(self):
"""
L{tree.addPyListings} accepts a document with nodes with their I{class}
attribute set to I{py-listing} and replaces those nodes with Python
source listings from the file given by the node's I{href} attribute.
"""
listingPath = FilePath(self.mktemp())
listingPath.setContent('def foo():\n pass\n')
parent = dom.Element('div')
listing = dom.Element('a')
listing.setAttribute('href', listingPath.basename())
listing.setAttribute('class', 'py-listing')
parent.appendChild(listing)
tree.addPyListings(parent, listingPath.dirname())
expected = """\
<div><div class="py-listing"><pre><p class="py-linenumber">1
2
</p><span class="py-src-keyword">def</span> <span class="py-src-identifier">foo</span>():
<span class="py-src-keyword">pass</span>
</pre><div class="caption"> - <a href="temp"><span class="filename">temp</span></a></div></div></div>"""
self.assertEqual(parent.toxml(), expected)
def pathEntryTree(self, tree):
"""
Create some files in a hierarchy, based on a dictionary describing those
files. The resulting hierarchy will be placed onto sys.path for the
duration of the test.
@param tree: A dictionary representing a directory structure. Keys are
strings, representing filenames, dictionary values represent
directories, string values represent file contents.
@return: another dictionary similar to the input, with file content
strings replaced with L{FilePath} objects pointing at where those
contents are now stored.
"""
def makeSomeFiles(pathobj, dirdict):
pathdict = {}
for (key, value) in dirdict.items():
child = pathobj.child(key)
if isinstance(value, bytes):
pathdict[key] = child
child.setContent(value)
elif isinstance(value, dict):
child.createDirectory()
pathdict[key] = makeSomeFiles(child, value)
else:
raise ValueError("only strings and dicts allowed as values")
return pathdict
base = FilePath(self.mktemp().encode("utf-8"))
base.makedirs()
result = makeSomeFiles(base, tree)
# On Python 3, sys.path cannot include byte paths:
self.replaceSysPath([base.path.decode("utf-8")] + sys.path)
self.replaceSysModules(sys.modules.copy())
return result
def test_doFile_withFilenameGenerator(self):
base = FilePath(self.mktemp())
base.makedirs()
def filenameGenerator(originalFileName, outputExtension):
name = os.path.splitext(FilePath(originalFileName).basename())[0]
return base.child(name + outputExtension).path
templ = dom.parse(open(d['template']))
tree.doFile(self.file, self.linkrel, d['ext'], d['baseurl'], templ, d, filenameGenerator)
self.assertXMLEqual(
"""\
<?xml version="1.0" ?><!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'><html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head><title>Twisted Documentation: My Test Lore Input</title></head>
<body bgcolor="white">
<h1 class="title">My Test Lore Input</h1>
<div class="content">
<span/>
<p>A Body.</p>
</div>
<a href="index.xhtml">Index</a>
</body>
</html>""",
base.child("simple.xhtml").getContent())
def test_getProcessor(self):
base = FilePath(self.mktemp())
base.makedirs()
input = base.child("simple3.html")
FilePath(__file__).sibling("simple3.html").copyTo(input)
options = { 'template': sp('template.tpl'), 'ext': '.xhtml', 'baseurl': 'burl',
'filenameMapping': None }
p = process.getProcessor(default, "html", options)
p(input.path, self.linkrel)
self.assertXMLEqual(
"""\
<?xml version="1.0" ?><!DOCTYPE html PUBLIC '-//W3C//DTD XHTML 1.0 Transitional//EN' 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd'><html lang="en" xmlns="http://www.w3.org/1999/xhtml">
<head><title>Twisted Documentation: My Test Lore Input</title></head>
<body bgcolor="white">
<h1 class="title">My Test Lore Input</h1>
<div class="content">
<span/>
<p>A Body.</p>
</div>
<a href="index.xhtml">Index</a>
</body>
</html>""",
base.child("simple3.xhtml").getContent())
def get_backend_api(test_case, cluster_id):
"""
Get an appropriate BackendAPI for the specified dataset backend.
Note this is a backdoor that is useful to be able to interact with cloud
APIs in tests. For many dataset backends this does not make sense, but it
provides a convenient means to interact with cloud backends such as EBS or
cinder.
:param test_case: The test case that is being run.
:param cluster_id: The unique cluster_id, used for backend APIs that
require this in order to be constructed.
"""
backend_config_filename = environ.get(
"FLOCKER_ACCEPTANCE_TEST_VOLUME_BACKEND_CONFIG")
if backend_config_filename is None:
raise SkipTest(
'This test requires the ability to construct an IBlockDeviceAPI '
'in order to verify construction. Please set '
'FLOCKER_ACCEPTANCE_TEST_VOLUME_BACKEND_CONFIG to a yaml filepath '
'with the dataset configuration.')
backend_name = environ.get("FLOCKER_ACCEPTANCE_VOLUME_BACKEND")
if backend_name is None:
raise SkipTest(
"Set acceptance testing volume backend using the " +
"FLOCKER_ACCEPTANCE_VOLUME_BACKEND environment variable.")
backend_config_filepath = FilePath(backend_config_filename)
full_backend_config = yaml.safe_load(
backend_config_filepath.getContent())
backend_config = full_backend_config.get(backend_name)
if 'backend' in backend_config:
backend_config.pop('backend')
backend = get_backend(backend_name)
return get_api(backend, pmap(backend_config), reactor, cluster_id)
def test_parseFileAndReportMismatchedTags(self):
"""
If the contents of the file passed to L{tree.parseFileAndReport}
contain a mismatched tag, L{process.ProcessingFailure} is raised
indicating the location of the open and close tags which were
mismatched.
"""
path = FilePath(self.mktemp())
path.setContent(' <foo>\n\n </bar>')
err = self.assertRaises(
process.ProcessingFailure, tree.parseFileAndReport, path.path)
self.assertEqual(
str(err),
"mismatched close tag at line 3, column 4; expected </foo> "
"(from line 1, column 2)")
# Test a case which requires involves proper close tag handling.
path.setContent('<foo><bar></bar>\n </baz>')
err = self.assertRaises(
process.ProcessingFailure, tree.parseFileAndReport, path.path)
self.assertEqual(
str(err),
"mismatched close tag at line 2, column 4; expected </foo> "
"(from line 1, column 0)")
def test_functional(self):
"""
It works
"""
root = FilePath(self.mktemp())
root.makedirs()
foo = root.child('foo')
foo.setContent('#!%s\n'
'import sys, os\n'
'print sys.argv[1]\n'
'print sys.stdin.read()\n' % (sys.executable,))
foo.chmod(0777)
history = []
runner = ScriptRunner(root.path, history.append)
r = runner.run('foo', ['something'], 'guts')
def check(proto):
self.assertEqual(proto.stdout, 'something\nguts\n')
self.assertEqual(proto.stderr, '')
self.assertTrue(len(history) > 0)
def eb(res):
print res
print ''
for i in history:
if i.key in [1,2,3]:
print i.data['line']
return res
return r.done.addCallback(check).addErrback(eb)
def copyPackage(title):
"""
Copy package directory to db path using a file lock to avoid potential
concurrency race conditions.
@param title: string to use in log entry
@type title: C{str}
"""
dbpath = FilePath(TimezoneCache.getDBPath())
pkgpath = TimezoneCache.FilteredFilePath(TimezoneCache._getPackageDBPath())
lockfile = FilesystemLock(dbpath.path + ".lock")
result = lockfile.lock()
try:
if result and not dbpath.exists():
log.info(
"{title} timezones from {pkg} to {to}",
title=title,
pkg=pkgpath.path,
to=dbpath.path
)
# Copy over the entire package
pkgpath.copyFilteredDirectoryTo(dbpath)
finally:
if result:
lockfile.unlock()
class VoluminousTests(TestCase):
def setUp(self):
self.tmpdir = FilePath(self.mktemp())
self.tmpdir.makedirs()
def test_create_volume(self):
dvol = VoluminousOptions()
dvol.parseOptions(ARGS + ["-p", self.tmpdir.path, "init", "foo"])
self.assertTrue(self.tmpdir.child("foo").exists())
self.assertTrue(self.tmpdir.child("foo").child("branches")
.child("master").exists())
self.assertEqual(dvol.voluminous.getOutput()[-1],
"Created volume foo\nCreated branch foo/master")
# Verify operation with `list`
dvol.parseOptions(ARGS + ["-p", self.tmpdir.path, "list"])
header, rest = self._parse_list_output(dvol)
expected_volumes = [["*", "foo", "master"]]
self.assertEqual(
sorted(expected_volumes),
sorted(rest),
)
def test_create_volume_already_exists(self):
dvol = VoluminousOptions()
# Create the repository twice, second time should have the error
expected_output = "Error: volume foo already exists"
dvol.parseOptions(ARGS + ["-p", self.tmpdir.path, "init", "foo"])
try:
dvol.parseOptions(ARGS + ["-p", self.tmpdir.path, "init", "foo"])
self.assertEqual(dvol.voluminous.getOutput()[-1], expected_output)
except CalledProcessErrorWithOutput, error:
self.assertIn(expected_output, error.original.output)
self.assertTrue(error.original.returncode != 0)
def postOptions(self):
if self['distribution'] is None:
raise UsageError("Distribution required.")
if self['config-file'] is not None:
config_file = FilePath(self['config-file'])
self['config'] = yaml.safe_load(config_file.getContent())
else:
self['config'] = {}
provider = self['provider'].lower()
provider_config = self['config'].get(provider, {})
package_source = PackageSource(
version=self['flocker-version'],
branch=self['branch'],
build_server=self['build-server'],
)
try:
get_runner = getattr(self, "_runner_" + provider.upper())
except AttributeError:
raise UsageError(
"Provider {!r} not supported. Available providers: {}".format(
provider, ', '.join(
name.lower() for name in self._get_provider_names()
)
)
)
else:
self.runner = get_runner(
package_source=package_source,
dataset_backend=self.dataset_backend(),
provider_config=provider_config,
)
def setUp(self):
# The "local" directory service
self.directory = DirectoryService(None)
# The "remote" directory service
if testMode == "xml":
# Need a copy as it might change
path = FilePath(os.path.join(os.path.dirname(__file__), "test.xml"))
copy = FilePath(self.mktemp())
path.copyTo(copy)
remoteDirectory = CalendarXMLDirectoryService(copy)
elif testMode == "od":
remoteDirectory = CalendarODDirectoryService()
# Connect the two services directly via an IOPump
client = AMP()
server = DirectoryProxyAMPProtocol(remoteDirectory)
pump = returnConnected(server, client)
# Replace the normal _getConnection method with one that bypasses any
# actual networking
self.patch(self.directory, "_getConnection", lambda: succeed(client))
# Wrap the normal _sendCommand method with one that flushes the IOPump
# afterwards
origCall = self.directory._sendCommand
def newCall(*args, **kwds):
d = origCall(*args, **kwds)
pump.flush()
return d
self.patch(self.directory, "_sendCommand", newCall)
def test_branch_multi_volumes(self, volumes):
"""
Always show the last checked-out branch for all volumes in ``list``.
"""
tmpdir = FilePath(self.mktemp())
tmpdir.makedirs()
dvol = VoluminousOptions()
for volume, branch in volumes:
dvol.parseOptions(ARGS + ["-p", tmpdir.path, "init", volume])
dvol.parseOptions(ARGS + ["-p", tmpdir.path, "commit", "-m", "hello"])
dvol.parseOptions(ARGS + ["-p", tmpdir.path, "checkout", "-b", branch])
dvol.parseOptions(ARGS + ["-p", tmpdir.path, "list"])
lines = dvol.voluminous.getOutput()[-1].split("\n")
header, rest = lines[0], lines[1:]
expected_volumes = [[volume, branch] for volume, branch in volumes]
# `init` activates the volume, so the last initialized volume is the
# active one.
expected_volumes[-1] = [
'*', expected_volumes[-1][0], expected_volumes[-1][1]]
self.assertEqual(['VOLUME', 'BRANCH', 'CONTAINERS'], header.split())
self.assertEqual(
sorted(expected_volumes),
sorted([line.split() for line in rest]),
)
def generate(cls, directory, control_hostname, num_nodes, cluster_id=None):
"""
Generate certificates in the given directory.
:param FilePath directory: Directory to use for certificate authority.
:param bytes control_hostname: The hostname of the control service.
:param int num_nodes: Number of nodes in the cluster.
:param UUID cluster_id: The unique identifier of the cluster for which
the certificates are being generated. If not given, a random
identifier will be generated.
:return: ``Certificates`` instance.
"""
RootCredential.initialize(
directory, b"acceptance-cluster", cluster_id=cluster_id,
)
def run(*arguments):
check_call([b"flocker-ca"] + list(arguments), cwd=directory.path)
run(b"create-control-certificate", control_hostname)
run(b"create-api-certificate", b"allison")
# Rename to user.crt/user.key so we can use this folder directly
# from flocker-deploy and other clients:
directory.child(b"allison.crt").moveTo(directory.child(b"user.crt"))
directory.child(b"allison.key").moveTo(directory.child(b"user.key"))
for i in range(num_nodes):
run(b"create-node-certificate")
for i, child in enumerate(
directory.globChildren(b"????????-????-*.crt")):
sibling = FilePath(child.path[:-3] + b"key")
child.moveTo(directory.child(b"node-%d.crt" % (i,)))
sibling.moveTo(directory.child(b"node-%d.key" % (i,)))
return cls(directory)
def setupJobdir(self):
jobdir = FilePath(self.mktemp())
jobdir.createDirectory()
self.jobdir = jobdir.path
for sub in 'new', 'tmp', 'cur':
jobdir.child(sub).createDirectory()
return self.jobdir
def setUp(self):
self.factory = OpenSSHFactory()
self.keysDir = FilePath(self.mktemp())
self.keysDir.makedirs()
self.factory.dataRoot = self.keysDir.path
self.moduliDir = FilePath(self.mktemp())
self.moduliDir.makedirs()
self.factory.moduliRoot = self.moduliDir.path
self.keysDir.child("ssh_host_foo").setContent(b"foo")
self.keysDir.child("bar_key").setContent(b"foo")
self.keysDir.child("ssh_host_one_key").setContent(
keydata.privateRSA_openssh)
self.keysDir.child("ssh_host_two_key").setContent(
keydata.privateDSA_openssh)
self.keysDir.child("ssh_host_three_key").setContent(
b"not a key content")
self.keysDir.child("ssh_host_one_key.pub").setContent(
keydata.publicRSA_openssh)
self.moduliDir.child("moduli").setContent(b"""
# $OpenBSD: moduli,v 1.xx 2016/07/26 12:34:56 jhacker Exp $
# Time Type Tests Tries Size Generator Modulus
20030501000000 2 6 100 2047 2 FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE45B3DC2007CB8A163BF0598DA48361C55D39A69163FA8FD24CF5F83655D23DCA3AD961C62F356208552BB9ED529077096966D670C354E4ABC9804F1746C08CA18217C32905E462E36CE3BE39E772C180E86039B2783A2EC07A28FB5C55DF06F4C52C9DE2BCBF6955817183995497CEA956AE515D2261898FA051015728E5A8AACAA68FFFFFFFFFFFFFFFF
19981111000000 2 6 100 1023 2 FFFFFFFFFFFFFFFFC90FDAA22168C234C4C6628B80DC1CD129024E088A67CC74020BBEA63B139B22514A08798E3404DDEF9519B3CD3A431B302B0A6DF25F14374FE1356D6D51C245E485B576625E7EC6F44C42E9A637ED6B0BFF5CB6F406B7EDEE386BFB5A899FA5AE9F24117C4B1FE649286651ECE65381FFFFFFFFFFFFFFFF
""")
self.mockos = MockOS()
self.patch(os, "seteuid", self.mockos.seteuid)
self.patch(os, "setegid", self.mockos.setegid)
def _get_device_path_virtio_blk(self, volume):
"""
The virtio_blk driver allows a serial number to be assigned to virtual
blockdevices.
OpenStack will set a serial number containing the first 20
characters of the Cinder block device ID.
This was introduced in
* https://github.com/openstack/nova/commit/3a47c02c58cefed0e230190b4bcef14527c82709 # noqa
* https://bugs.launchpad.net/nova/+bug/1004328
The udev daemon will read the serial number and create a
symlink to the canonical virtio_blk device path.
We do this because libvirt does not return the correct device path when
additional disks have been attached using a client other than
cinder. This is expected behaviour within Cinder and libvirt See
https://bugs.launchpad.net/cinder/+bug/1387945 and
http://libvirt.org/formatdomain.html#elementsDisks (target section)
:param volume: The Cinder ``Volume`` which is attached.
:returns: ``FilePath`` of the device created by the virtio_blk
driver.
"""
expected_path = FilePath(
"/dev/disk/by-id/virtio-{}".format(volume.id[:20])
)
if expected_path.exists():
return expected_path.realpath()
else:
raise UnattachedVolume(volume.id)
def test_functional_centos_7(self):
"""
The expected RPM files are built for CentOS 7
"""
output_dir = FilePath(self.mktemp())
check_call([
FLOCKER_PATH.descendant([b'admin', b'build-package']).path,
'--destination-path', output_dir.path,
'--distribution', 'centos-7',
FLOCKER_PATH.path
])
python_version = __version__
rpm_version = make_rpm_version(python_version)
expected_basenames = (
('clusterhq-python-flocker', 'x86_64'),
('clusterhq-flocker-cli', 'noarch'),
('clusterhq-flocker-node', 'noarch'),
)
expected_filenames = []
for basename, arch in expected_basenames:
f = '{}-{}-{}.{}.rpm'.format(
basename, rpm_version.version, rpm_version.release, arch)
expected_filenames.append(f)
self.assertEqual(
set(expected_filenames),
set(f.basename() for f in output_dir.children())
)
for f in output_dir.children():
assert_rpm_lint(self, f)
def test_bootstrap_pyc(self):
"""
``create_virtualenv`` creates links to the pyc files for all the
modules required for the virtualenv bootstrap process.
"""
target_path = FilePath(self.mktemp())
create_virtualenv(root=target_path)
py_files = []
for module_name in VIRTUALENV_REQUIRED_MODULES:
py_base = target_path.descendant(['lib', 'python2.7', module_name])
py = py_base.siblingExtension('.py')
pyc = py_base.siblingExtension('.pyc')
if py.exists() and False in (py.islink(), pyc.islink()):
py_files.append('PY: {} > {}\nPYC: {} > {}\n'.format(
'/'.join(py.segmentsFrom(target_path)),
py.realpath().path,
'/'.join(pyc.segmentsFrom(target_path)),
pyc.islink() and pyc.realpath().path or 'NOT A SYMLINK'
))
if py_files:
self.fail(
'Non-linked bootstrap pyc files in {}: \n{}'.format(
target_path, '\n'.join(py_files)
)
)
def flocker_deploy(test_case, deployment_config, application_config):
"""
Run ``flocker-deploy`` with given configuration files.
:param test_case: The ``TestCase`` running this unit test.
:param dict deployment_config: The desired deployment configuration.
:param dict application_config: The desired application configuration.
"""
# This is duplicate code, see
# https://clusterhq.atlassian.net/browse/FLOC-1903
control_node = environ.get("FLOCKER_ACCEPTANCE_CONTROL_NODE")
certificate_path = environ["FLOCKER_ACCEPTANCE_API_CERTIFICATES_PATH"]
if control_node is None:
raise SkipTest("Set control node address using "
"FLOCKER_ACCEPTANCE_CONTROL_NODE environment variable.")
temp = FilePath(test_case.mktemp())
temp.makedirs()
deployment = temp.child(b"deployment.yml")
deployment.setContent(safe_dump(deployment_config))
application = temp.child(b"application.yml")
application.setContent(safe_dump(application_config))
check_call([b"flocker-deploy", b"--certificates-directory",
certificate_path, control_node, deployment.path,
application.path])
def connect_worker(self, case):
if RemoteWorker is None:
raise SkipTest("buildbot-worker package is not installed")
workdir = FilePath(case.mktemp())
workdir.createDirectory()
self.remote_worker = RemoteWorker(self.worker.name, workdir.path, False)
self.remote_worker.setServiceParent(self.worker)
class Tests(SynchronousTestCase):
def setUp(self):
self.options = options_type()
self.scratch_directory = FilePath(self.mktemp())
self.scratch_directory.makedirs()
self.sample_content = yaml.safe_dump({
u"control-service": {
u"hostname": u"10.0.0.1",
u"port": 4524,
},
u"version": 1,
})
self.config = self.scratch_directory.child('dataset-config.yml')
self.config.setContent(self.sample_content)
def test_default_config_file(self):
"""
The default config file is a FilePath with path
``/etc/flocker/agent.yml``.
"""
self.options.parseOptions([])
self.assertEqual(
self.options["agent-config"],
FilePath("/etc/flocker/agent.yml"))
def test_custom_config_file(self):
"""
The ``--config-file`` command-line option allows configuring
the config file.
"""
self.options.parseOptions(
[b"--agent-config", b"/etc/foo.yml"])
self.assertEqual(
self.options["agent-config"],
FilePath("/etc/foo.yml"))
def reportEnvironment():
revision = twisted.version.short().split('r', 1)[1]
packageDirectory = FilePath(twisted.__file__).parent()
try:
import pysvn
except ImportError:
entries = packageDirectory.child('.svn').child('entries').getContent()
lines = entries.splitlines()
revision = lines[3]
date = lines[9][:20].replace('T', ' ')
else:
client = pysvn.Client()
[entry] = client.log(
packageDirectory.path,
pysvn.Revision(pysvn.opt_revision_kind.number, int(revision)),
limit=1)
date = str(datetime.fromtimestamp(entry['date']))
return {
'project': 'Twisted',
'executable': executable,
'environment': uname()[1],
'commitid': revision,
'revision_date': date,
'result_date': str(datetime.now()),
}
def test_internal_symlinks_only(self):
"""
The resulting ``virtualenv`` only contains symlinks to files inside the
virtualenv and to /usr on the host OS.
"""
target_path = FilePath(self.mktemp())
create_virtualenv(root=target_path)
allowed_targets = (target_path, FilePath('/usr'),)
bad_links = []
for path in target_path.walk():
if path.islink():
realpath = path.realpath()
for allowed_target in allowed_targets:
try:
realpath.segmentsFrom(allowed_target)
except ValueError:
pass
else:
# The target is a descendent of an allowed_target.
break
else:
bad_links.append(path)
if bad_links:
self.fail(
"Symlinks outside of virtualenv detected:" +
'\n'.join(
'/'.join(
path.segmentsFrom(target_path)
) + ' -> ' + path.realpath().path
for path in bad_links
)
)
def test_load_error_on_unreadable_key_file(self):
"""
A ``PathError`` is raised if the key file path given to
``RootCredential.from_path`` cannot be opened for reading.
"""
path = FilePath(self.mktemp())
path.makedirs()
crt_path = path.child(AUTHORITY_CERTIFICATE_FILENAME)
crt_file = crt_path.open(b'w')
crt_file.write(b"dummy")
crt_file.close()
key_path = path.child(AUTHORITY_KEY_FILENAME)
key_file = key_path.open(b'w')
key_file.write(b"dummy")
key_file.close()
# make file unreadable
key_path.chmod(0o100)
e = self.assertRaises(
PathError, RootCredential.from_path, path
)
expected = (
"Unable to load certificate authority file. "
"Permission denied {path}"
).format(path=key_path.path)
self.assertEqual(str(e), expected)
def test_functional_ubuntu_1404(self):
"""
The expected deb files are generated on Ubuntu14.04.
"""
output_dir = FilePath(self.mktemp())
check_call([
FLOCKER_PATH.descendant([b'admin', b'build-package']).path,
'--destination-path', output_dir.path,
'--distribution', 'ubuntu-14.04',
FLOCKER_PATH.path
])
python_version = __version__
rpm_version = make_rpm_version(python_version)
expected_basenames = (
('clusterhq-python-flocker', 'amd64'),
('clusterhq-flocker-cli', 'all'),
('clusterhq-flocker-node', 'all'),
)
expected_filenames = []
for basename, arch in expected_basenames:
f = '{}_{}-{}_{}.deb'.format(
basename, rpm_version.version, rpm_version.release, arch)
expected_filenames.append(f)
self.assertEqual(
set(expected_filenames),
set(f.basename() for f in output_dir.children())
)
for f in output_dir.children():
assert_deb_lint(self, f)
def setUp(self):
self.tmpDir = FilePath(self.mktemp())
def test_repr(self):
"""
The representation of a Project is Project(directory).
"""
foo = Project(FilePath('bar'))
self.assertEqual(repr(foo), 'Project(%r)' % (foo.directory))
class ProcessTestsBuilder(ProcessTestsBuilderBase):
"""
Builder defining tests relating to L{IReactorProcess} for child processes
which do not have a PTY.
"""
usePTY = False
keepStdioOpenProgram = FilePath(__file__).sibling('process_helper.py').path
if platform.isWindows():
keepStdioOpenArg = "windows"
else:
# Just a value that doesn't equal "windows"
keepStdioOpenArg = ""
# Define this test here because PTY-using processes only have stdin and
# stdout and the test would need to be different for that to work.
def test_childConnectionLost(self):
"""
L{IProcessProtocol.childConnectionLost} is called each time a file
descriptor associated with a child process is closed.
"""
connected = Deferred()
lost = {0: Deferred(), 1: Deferred(), 2: Deferred()}
class Closer(ProcessProtocol):
def makeConnection(self, transport):
connected.callback(transport)
def childConnectionLost(self, childFD):
lost[childFD].callback(None)
source = ("import os, sys\n"
"while 1:\n"
" line = sys.stdin.readline().strip()\n"
" if not line:\n"
" break\n"
" os.close(int(line))\n")
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Closer(),
sys.executable, [sys.executable, "-c", source],
usePTY=self.usePTY)
def cbConnected(transport):
transport.write('2\n')
return lost[2].addCallback(lambda ign: transport)
connected.addCallback(cbConnected)
def lostSecond(transport):
transport.write('1\n')
return lost[1].addCallback(lambda ign: transport)
connected.addCallback(lostSecond)
def lostFirst(transport):
transport.write('\n')
connected.addCallback(lostFirst)
connected.addErrback(err)
def cbEnded(ignored):
reactor.stop()
connected.addCallback(cbEnded)
self.runReactor(reactor)
# This test is here because PTYProcess never delivers childConnectionLost.
def test_processEnded(self):
"""
L{IProcessProtocol.processEnded} is called after the child process
exits and L{IProcessProtocol.childConnectionLost} is called for each of
its file descriptors.
"""
ended = Deferred()
lost = []
class Ender(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
self.transport.loseConnection()
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD, ))
lost.append(childFD)
def processExited(self, reason):
msg('processExited(%r)' % (reason, ))
def processEnded(self, reason):
msg('processEnded(%r)' % (reason, ))
ended.callback([reason])
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Ender(),
sys.executable, [
sys.executable, self.keepStdioOpenProgram,
"child", self.keepStdioOpenArg
],
usePTY=self.usePTY)
def cbEnded((failure, )):
failure.trap(ProcessDone)
self.assertEqual(set(lost), set([0, 1, 2]))
ended.addCallback(cbEnded)
ended.addErrback(err)
ended.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
# This test is here because PTYProcess.loseConnection does not actually
# close the file descriptors to the child process. This test needs to be
# written fairly differently for PTYProcess.
def test_processExited(self):
"""
L{IProcessProtocol.processExited} is called when the child process
exits, even if file descriptors associated with the child are still
open.
"""
exited = Deferred()
allLost = Deferred()
lost = []
class Waiter(ProcessProtocol):
def childDataReceived(self, fd, data):
msg('childDataReceived(%d, %r)' % (fd, data))
def childConnectionLost(self, childFD):
msg('childConnectionLost(%d)' % (childFD, ))
lost.append(childFD)
if len(lost) == 3:
allLost.callback(None)
def processExited(self, reason):
msg('processExited(%r)' % (reason, ))
# See test_processExitedWithSignal
exited.callback([reason])
self.transport.loseConnection()
reactor = self.buildReactor()
reactor.callWhenRunning(reactor.spawnProcess,
Waiter(),
sys.executable, [
sys.executable, self.keepStdioOpenProgram,
"child", self.keepStdioOpenArg
],
usePTY=self.usePTY)
def cbExited((failure, )):
failure.trap(ProcessDone)
msg('cbExited; lost = %s' % (lost, ))
self.assertEqual(lost, [])
return allLost
exited.addCallback(cbExited)
def cbAllLost(ignored):
self.assertEqual(set(lost), set([0, 1, 2]))
exited.addCallback(cbAllLost)
exited.addErrback(err)
exited.addCallback(lambda ign: reactor.stop())
self.runReactor(reactor)
def makeSourceFile(self, sourceLines):
"""
Write the given list of lines to a text file and return the absolute
path to it.
"""
script = self.mktemp()
scriptFile = file(script, 'wt')
scriptFile.write(os.linesep.join(sourceLines) + os.linesep)
scriptFile.close()
return os.path.abspath(script)
def test_shebang(self):
"""
Spawning a process with an executable which is a script starting
with an interpreter definition line (#!) uses that interpreter to
evaluate the script.
"""
SHEBANG_OUTPUT = 'this is the shebang output'
scriptFile = self.makeSourceFile([
"#!%s" % (sys.executable, ), "import sys",
"sys.stdout.write('%s')" % (SHEBANG_OUTPUT, ), "sys.stdout.flush()"
])
os.chmod(scriptFile, 0700)
reactor = self.buildReactor()
def cbProcessExited((out, err, code)):
msg("cbProcessExited((%r, %r, %d))" % (out, err, code))
self.assertEqual(out, SHEBANG_OUTPUT)
self.assertEqual(err, "")
self.assertEqual(code, 0)
def shutdown(passthrough):
reactor.stop()
return passthrough
def start():
d = utils.getProcessOutputAndValue(scriptFile, reactor=reactor)
d.addBoth(shutdown)
d.addCallback(cbProcessExited)
d.addErrback(err)
reactor.callWhenRunning(start)
self.runReactor(reactor)
def test_processCommandLineArguments(self):
"""
Arguments given to spawnProcess are passed to the child process as
originally intended.
"""
source = (
# On Windows, stdout is not opened in binary mode by default,
# so newline characters are munged on writing, interfering with
# the tests.
'import sys, os\n'
'try:\n'
' import msvcrt\n'
' msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)\n'
'except ImportError:\n'
' pass\n'
'for arg in sys.argv[1:]:\n'
' sys.stdout.write(arg + chr(0))\n'
' sys.stdout.flush()')
args = ['hello', '"', ' \t|<>^&', r'"\\"hello\\"', r'"foo\ bar baz\""']
# Ensure that all non-NUL characters can be passed too.
args.append(''.join(map(chr, xrange(1, 256))))
reactor = self.buildReactor()
def processFinished(output):
output = output.split('\0')
# Drop the trailing \0.
output.pop()
self.assertEqual(args, output)
def shutdown(result):
reactor.stop()
return result
def spawnChild():
d = succeed(None)
d.addCallback(lambda dummy: utils.getProcessOutput(
sys.executable, ['-c', source] + args, reactor=reactor))
d.addCallback(processFinished)
d.addBoth(shutdown)
reactor.callWhenRunning(spawnChild)
self.runReactor(reactor)
from twisted.web import resource
from twisted.python.filepath import FilePath
from webmagic.untwist import BetterResource, BetterFile, ConnectionTrackingSite
from webmagic.special import WaitResource
import coreweb
here = FilePath(coreweb.__path__[0])
class CachedFile(BetterFile):
isLeaf = True
def render_GET(self, request):
request.responseHeaders.setRawHeaders(
'Expires', ['Sat, 30 Dec 2034 16:00:00 GMT'])
request.responseHeaders.setRawHeaders('Cache-Control',
['max-age=99999999'])
return BetterFile.render_GET(self, request)
class Root(BetterResource):
def __init__(self, reactor, closureLibrary):
import js_coreweb
resource.Resource.__init__(self)
self.putChild('', BetterFile(here.child('index.html').path))
self.putChild('js_coreweb_tests.html',
BetterFile(here.child('js_coreweb_tests.html').path))
def buildDirectory(
store,
dataRoot,
servicesInfo,
augmentServiceInfo,
wikiServiceInfo,
serversDB=None,
cachingSeconds=0,
filterStartsWith=False,
lookupsBetweenPurges=0,
negativeCaching=True,
):
"""
Return a directory without using a config object; suitable for tests
which need to have mulitple directory instances.
@param store: The store.
@param dataRoot: The path to the directory containing xml files for any xml
based services.
@param servicesInfo: An interable of ConfigDicts mirroring the
DirectoryService and ResourceService sections of stdconfig
@param augmentServiceInfo: A ConfigDict mirroring the AugmentService section
of stdconfig
@param wikiServiceInfo: A ConfigDict mirroring the Wiki section of stdconfig
@param serversDB: A ServersDB object to assign to the directory
"""
aggregatedServices = []
cachingServices = []
ldapService = None # LDAP DS has extra stats (see augment.py)
for serviceValue in servicesInfo:
if not serviceValue.Enabled:
continue
directoryType = serviceValue.type.lower()
params = serviceValue.params
if "xml" in directoryType:
xmlFile = params.xmlFile
xmlFile = fullServerPath(dataRoot, xmlFile)
fp = FilePath(xmlFile)
if not fp.exists():
fp.setContent(DEFAULT_XML_CONTENT)
directory = XMLDirectoryService(fp)
elif "opendirectory" in directoryType:
from txdav.who.opendirectory import (DirectoryService as
ODDirectoryService)
# We don't want system accounts returned in lookups, so tell
# the service to suppress them.
node = params.node
directory = ODDirectoryService(nodeName=node,
suppressSystemRecords=True)
elif "ldap" in directoryType:
from twext.who.ldap import (DirectoryService as
LDAPDirectoryService, FieldName as
LDAPFieldName, RecordTypeSchema)
if params.credentials.dn and params.credentials.password:
creds = UsernamePassword(params.credentials.dn,
params.credentials.password)
else:
creds = None
mapping = params.mapping
extraFilters = params.extraFilters
directory = LDAPDirectoryService(
params.uri,
params.rdnSchema.base,
useTLS=params.useTLS,
credentials=creds,
fieldNameToAttributesMap=MappingProxyType({
BaseFieldName.uid:
mapping.uid,
BaseFieldName.guid:
mapping.guid,
BaseFieldName.shortNames:
mapping.shortNames,
BaseFieldName.fullNames:
mapping.fullNames,
BaseFieldName.emailAddresses:
mapping.emailAddresses,
LDAPFieldName.memberDNs:
mapping.memberDNs,
CalFieldName.readOnlyProxy:
mapping.readOnlyProxy,
CalFieldName.readWriteProxy:
mapping.readWriteProxy,
CalFieldName.loginAllowed:
mapping.loginAllowed,
CalFieldName.hasCalendars:
mapping.hasCalendars,
CalFieldName.autoScheduleMode:
mapping.autoScheduleMode,
CalFieldName.autoAcceptGroup:
mapping.autoAcceptGroup,
CalFieldName.serviceNodeUID:
mapping.serviceNodeUID,
CalFieldName.associatedAddress:
mapping.associatedAddress,
CalFieldName.geographicLocation:
mapping.geographicLocation,
CalFieldName.streetAddress:
mapping.streetAddress,
}),
recordTypeSchemas=MappingProxyType({
RecordType.user:
RecordTypeSchema(
relativeDN=params.rdnSchema.users,
attributes=(),
),
RecordType.group:
RecordTypeSchema(
relativeDN=params.rdnSchema.groups,
attributes=(),
),
CalRecordType.location:
RecordTypeSchema(
relativeDN=params.rdnSchema.locations,
attributes=(),
),
CalRecordType.resource:
RecordTypeSchema(
relativeDN=params.rdnSchema.resources,
attributes=(),
),
CalRecordType.address:
RecordTypeSchema(
relativeDN=params.rdnSchema.addresses,
attributes=(),
),
}),
extraFilters={
RecordType.user: extraFilters.get("users", ""),
RecordType.group: extraFilters.get("groups", ""),
CalRecordType.location: extraFilters.get("locations", ""),
CalRecordType.resource: extraFilters.get("resources", ""),
CalRecordType.address: extraFilters.get("addresses", ""),
},
threadPoolMax=params.get("threadPoolMax", 10),
authConnectionMax=params.get("authConnectionMax", 5),
queryConnectionMax=params.get("queryConnectionMax", 5),
tries=params.get("tries", 3),
warningThresholdSeconds=params.get("warningThresholdSeconds",
5),
)
ldapService = directory
elif "inmemory" in directoryType:
from txdav.who.test.support import CalendarInMemoryDirectoryService
directory = CalendarInMemoryDirectoryService()
else:
log.error("Invalid DirectoryType: {dt}", dt=directoryType)
raise DirectoryConfigurationError
# Set the appropriate record types on each service
types = []
fieldNames = []
for recordTypeName in params.recordTypes:
recordType = {
"users": RecordType.user,
"groups": RecordType.group,
"locations": CalRecordType.location,
"resources": CalRecordType.resource,
"addresses": CalRecordType.address,
}.get(recordTypeName, None)
if recordType is None:
log.error("Invalid Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
if recordType in types:
log.error("Duplicate Record Type: {rt}", rt=recordTypeName)
raise DirectoryConfigurationError
types.append(recordType)
directory.recordType = ConstantsContainer(types)
directory.fieldName = ConstantsContainer(
(directory.fieldName, CalFieldName))
fieldNames.append(directory.fieldName)
if cachingSeconds:
directory = CachingDirectoryService(
directory,
expireSeconds=cachingSeconds,
lookupsBetweenPurges=lookupsBetweenPurges,
negativeCaching=negativeCaching,
)
cachingServices.append(directory)
aggregatedServices.append(directory)
#
# Setup the Augment Service
#
serviceClass = {
"xml": "twistedcaldav.directory.augment.AugmentXMLDB",
}
for augmentFile in augmentServiceInfo.params.xmlFiles:
augmentFile = fullServerPath(dataRoot, augmentFile)
augmentFilePath = FilePath(augmentFile)
if not augmentFilePath.exists():
augmentFilePath.setContent(DEFAULT_AUGMENT_CONTENT)
augmentClass = namedClass(serviceClass[augmentServiceInfo.type])
log.info("Configuring augment service of type: {augmentClass}",
augmentClass=augmentClass)
try:
augmentService = augmentClass(**augmentServiceInfo.params)
except IOError:
log.error("Could not start augment service")
raise
userDirectory = None
for directory in aggregatedServices:
if RecordType.user in directory.recordTypes():
userDirectory = directory
break
else:
log.error("No directory service set up for users")
raise DirectoryConfigurationError
# Delegate service
delegateDirectory = DelegateDirectoryService(userDirectory.realmName,
store)
# (put at front of list so we don't try to ask the actual DS services
# about the delegate-related principals, for performance)
aggregatedServices.insert(0, delegateDirectory)
# Wiki service
if wikiServiceInfo.Enabled:
aggregatedServices.append(
WikiDirectoryService(
userDirectory.realmName,
wikiServiceInfo.EndpointDescriptor,
))
# Aggregate service
aggregateDirectory = AggregateDirectoryService(userDirectory.realmName,
aggregatedServices)
# Augment service
try:
fieldNames.append(CalFieldName)
augmented = AugmentedDirectoryService(aggregateDirectory, store,
augmentService)
augmented.fieldName = ConstantsContainer(fieldNames)
# The delegate directory needs a way to look up user/group records
# so hand it a reference to the augmented directory.
# FIXME: is there a better pattern to use here?
delegateDirectory.setMasterDirectory(augmented)
# Tell each caching service what method to use when reporting
# times and cache stats
for cachingService in cachingServices:
cachingService.setTimingMethod(augmented._addTiming)
# LDAP has additional stats to report
augmented._ldapDS = ldapService
except Exception as e:
log.error("Could not create directory service", error=e)
raise
if serversDB is not None:
augmented.setServersDB(serversDB)
if filterStartsWith:
augmented.setFilter(startswithFilter)
return augmented
class CheckNewsfragmentScriptTests(ExternalTempdirTestCase):
"""
L{CheckNewsfragmentScript}.
"""
def setUp(self):
self.origin = FilePath(self.mktemp())
_gitInit(self.origin)
runCommand(["git", "checkout", "-b", "trunk"], cwd=self.origin.path)
self.origin.child("test").setContent(b"test!")
runCommand(["git", "add", self.origin.child("test").path], cwd=self.origin.path)
runCommand(["git", "commit", "-m", "initial"], cwd=self.origin.path)
self.repo = FilePath(self.mktemp())
runCommand(["git", "clone", self.origin.path, self.repo.path])
_gitConfig(self.repo)
def test_noArgs(self):
"""
Too few arguments returns a failure.
"""
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([])
self.assertEqual(
e.exception.args, ("Must specify one argument: the Twisted checkout",)
)
def test_diffFromTrunkNoNewsfragments(self):
"""
If there are changes from trunk, then there should also be a
newsfragment.
"""
runCommand(["git", "checkout", "-b", "mypatch"], cwd=self.repo.path)
somefile = self.repo.child("somefile")
somefile.setContent(b"change")
runCommand(["git", "add", somefile.path, somefile.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "some file"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (1,))
self.assertEqual(logs[-1], "No newsfragment found. Have you committed it?")
def test_noChangeFromTrunk(self):
"""
If there are no changes from trunk, then no need to check the
newsfragments
"""
runCommand(["git", "checkout", "-b", "mypatch"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (0,))
self.assertEqual(
logs[-1], "On trunk or no diffs from trunk; no need to look at this."
)
def test_trunk(self):
"""
Running it on trunk always gives green.
"""
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (0,))
self.assertEqual(
logs[-1], "On trunk or no diffs from trunk; no need to look at this."
)
def test_release(self):
"""
Running it on a release branch returns green if there is no
newsfragments even if there are changes.
"""
runCommand(
["git", "checkout", "-b", "release-16.11111-9001"], cwd=self.repo.path
)
somefile = self.repo.child("somefile")
somefile.setContent(b"change")
runCommand(["git", "add", somefile.path, somefile.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "some file"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (0,))
self.assertEqual(logs[-1], "Release branch with no newsfragments, all good.")
def test_releaseWithNewsfragments(self):
"""
Running it on a release branch returns red if there are new
newsfragments.
"""
runCommand(
["git", "checkout", "-b", "release-16.11111-9001"], cwd=self.repo.path
)
newsfragments = self.repo.child("twisted").child("newsfragments")
newsfragments.makedirs()
fragment = newsfragments.child("1234.misc")
fragment.setContent(b"")
unrelated = self.repo.child("somefile")
unrelated.setContent(b"Boo")
runCommand(["git", "add", fragment.path, unrelated.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "fragment"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (1,))
self.assertEqual(logs[-1], "No newsfragments should be on the release branch.")
def test_onlyQuotes(self):
"""
Running it on a branch with only a quotefile change gives green.
"""
runCommand(["git", "checkout", "-b", "quotefile"], cwd=self.repo.path)
fun = self.repo.child("docs").child("fun")
fun.makedirs()
quotes = fun.child("Twisted.Quotes")
quotes.setContent(b"Beep boop")
runCommand(["git", "add", quotes.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "quotes"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (0,))
self.assertEqual(logs[-1], "Quotes change only; no newsfragment needed.")
def test_newsfragmentAdded(self):
"""
Running it on a branch with a fragment in the newsfragments dir added
returns green.
"""
runCommand(["git", "checkout", "-b", "quotefile"], cwd=self.repo.path)
newsfragments = self.repo.child("twisted").child("newsfragments")
newsfragments.makedirs()
fragment = newsfragments.child("1234.misc")
fragment.setContent(b"")
unrelated = self.repo.child("somefile")
unrelated.setContent(b"Boo")
runCommand(["git", "add", fragment.path, unrelated.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "newsfragment"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (0,))
self.assertEqual(logs[-1], "Found twisted/newsfragments/1234.misc")
def test_topfileButNotFragmentAdded(self):
"""
Running it on a branch with a non-fragment in the topfiles dir does not
return green.
"""
runCommand(["git", "checkout", "-b", "quotefile"], cwd=self.repo.path)
topfiles = self.repo.child("twisted").child("topfiles")
topfiles.makedirs()
notFragment = topfiles.child("1234.txt")
notFragment.setContent(b"")
unrelated = self.repo.child("somefile")
unrelated.setContent(b"Boo")
runCommand(["git", "add", notFragment.path, unrelated.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "not topfile"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (1,))
self.assertEqual(logs[-1], "No newsfragment found. Have you committed it?")
def test_newsfragmentAddedButWithOtherNewsfragments(self):
"""
Running it on a branch with a fragment in the topfiles dir added
returns green, even if there are other files in the topfiles dir.
"""
runCommand(["git", "checkout", "-b", "quotefile"], cwd=self.repo.path)
newsfragments = self.repo.child("twisted").child("newsfragments")
newsfragments.makedirs()
fragment = newsfragments.child("1234.misc")
fragment.setContent(b"")
unrelated = newsfragments.child("somefile")
unrelated.setContent(b"Boo")
runCommand(["git", "add", fragment.path, unrelated.path], cwd=self.repo.path)
runCommand(["git", "commit", "-m", "newsfragment"], cwd=self.repo.path)
logs = []
with self.assertRaises(SystemExit) as e:
CheckNewsfragmentScript(logs.append).main([self.repo.path])
self.assertEqual(e.exception.args, (0,))
self.assertEqual(logs[-1], "Found twisted/newsfragments/1234.misc")
def setUp(self):
self.repos = FilePath(self.mktemp())
def test_buildWithDeprecated(self):
"""
The templates and System for Twisted includes adding deprecations.
"""
stdout = BytesIO()
self.patch(sys, "stdout", stdout)
projectName = "Foobar"
packageName = "quux"
projectURL = "scheme:project"
sourceURL = "scheme:source"
docstring = "text in docstring"
privateDocstring = "should also appear in output"
inputPath = FilePath(self.mktemp()).child(packageName)
inputPath.makedirs()
inputPath.child("__init__.py").setContent(
"from twisted.python.deprecate import deprecated\n"
"from incremental import Version\n"
"@deprecated(Version('Twisted', 15, 0, 0), "
"'Baz')\n"
"def foo():\n"
" '{}'\n"
"from twisted.python import deprecate\n"
"import incremental\n"
"@deprecate.deprecated(incremental.Version('Twisted', 16, 0, 0))\n"
"def _bar():\n"
" '{}'\n"
"@deprecated(Version('Twisted', 14, 2, 3), replacement='stuff')\n"
"class Baz:\n"
" pass"
"".format(docstring, privateDocstring).encode()
)
outputPath = FilePath(self.mktemp())
builder = APIBuilder()
builder.build(projectName, projectURL, sourceURL, inputPath, outputPath)
quuxPath = outputPath.child("quux.html")
self.assertTrue(
quuxPath.exists(),
f"Package documentation file {quuxPath.path!r} did not exist.",
)
self.assertIn(
docstring,
quuxPath.getContent().decode(),
"Docstring not in package documentation file.",
)
self.assertIn(
"foo was deprecated in Twisted 15.0.0; please use Baz instead.",
quuxPath.getContent().decode(),
)
self.assertIn(
"_bar was deprecated in Twisted 16.0.0.", quuxPath.getContent().decode()
)
self.assertIn(privateDocstring, quuxPath.getContent().decode())
self.assertIn(
"Baz was deprecated in Twisted 14.2.3; please use stuff instead.",
quuxPath.sibling("quux.Baz.html").getContent().decode(),
)
self.assertEqual(stdout.getvalue(), b"")
def make_tftp_root(self):
"""Set, and return, a temporary TFTP root directory."""
tftproot = self.make_dir()
self.useFixture(ClusterConfigurationFixture(tftp_root=tftproot))
return FilePath(tftproot)
class SphinxBuilderTests(TestCase):
"""
Tests for L{SphinxBuilder}.
@note: This test case depends on twisted.web, which violates the standard
Twisted practice of not having anything in twisted.python depend on
other Twisted packages and opens up the possibility of creating
circular dependencies. Do not use this as an example of how to
structure your dependencies.
@ivar builder: A plain L{SphinxBuilder}.
@ivar sphinxDir: A L{FilePath} representing a directory to be used for
containing a Sphinx project.
@ivar sourceDir: A L{FilePath} representing a directory to be used for
containing the source files for a Sphinx project.
"""
confContent = """\
source_suffix = '.rst'
master_doc = 'index'
"""
confContent = textwrap.dedent(confContent)
indexContent = """\
==============
This is a Test
==============
This is only a test
-------------------
In case you hadn't figured it out yet, this is a test.
"""
indexContent = textwrap.dedent(indexContent)
def setUp(self):
"""
Set up a few instance variables that will be useful.
"""
self.builder = SphinxBuilder()
# set up a place for a fake sphinx project
self.twistedRootDir = FilePath(self.mktemp())
self.sphinxDir = self.twistedRootDir.child("docs")
self.sphinxDir.makedirs()
self.sourceDir = self.sphinxDir
def createFakeSphinxProject(self):
"""
Create a fake Sphinx project for test purposes.
Creates a fake Sphinx project with the absolute minimum of source
files. This includes a single source file ('index.rst') and the
smallest 'conf.py' file possible in order to find that source file.
"""
self.sourceDir.child("conf.py").setContent(self.confContent.encode())
self.sourceDir.child("index.rst").setContent(self.indexContent.encode())
def verifyFileExists(self, fileDir, fileName):
"""
Helper which verifies that C{fileName} exists in C{fileDir} and it has
some content.
@param fileDir: A path to a directory.
@type fileDir: L{FilePath}
@param fileName: The last path segment of a file which may exist within
C{fileDir}.
@type fileName: L{str}
@raise FailTest: If C{fileDir.child(fileName)}:
1. Does not exist.
2. Is empty.
3. In the case where it's a path to a C{.html} file, the
content looks like an HTML file.
@return: L{None}
"""
# check that file exists
fpath = fileDir.child(fileName)
self.assertTrue(fpath.exists())
# check that the output files have some content
fcontents = fpath.getContent()
self.assertTrue(len(fcontents) > 0)
# check that the html files are at least html-ish
# this is not a terribly rigorous check
if fpath.path.endswith(".html"):
self.assertIn(b"<body", fcontents)
def test_build(self):
"""
Creates and builds a fake Sphinx project using a L{SphinxBuilder}.
"""
self.createFakeSphinxProject()
self.builder.build(self.sphinxDir)
self.verifyBuilt()
def test_main(self):
"""
Creates and builds a fake Sphinx project as if via the command line.
"""
self.createFakeSphinxProject()
self.builder.main([self.sphinxDir.parent().path])
self.verifyBuilt()
def test_warningsAreErrors(self):
"""
Creates and builds a fake Sphinx project as if via the command line,
failing if there are any warnings.
"""
output = StringIO()
self.patch(sys, "stdout", output)
self.createFakeSphinxProject()
with self.sphinxDir.child("index.rst").open("a") as f:
f.write(b"\n.. _malformed-link-target\n")
exception = self.assertRaises(
SystemExit, self.builder.main, [self.sphinxDir.parent().path]
)
self.assertEqual(exception.code, 1)
self.assertIn("malformed hyperlink target", output.getvalue())
self.verifyBuilt()
def verifyBuilt(self):
"""
Verify that a sphinx project has been built.
"""
htmlDir = self.sphinxDir.sibling("doc")
self.assertTrue(htmlDir.isdir())
doctreeDir = htmlDir.child("doctrees")
self.assertFalse(doctreeDir.exists())
self.verifyFileExists(htmlDir, "index.html")
self.verifyFileExists(htmlDir, "genindex.html")
self.verifyFileExists(htmlDir, "objects.inv")
self.verifyFileExists(htmlDir, "search.html")
self.verifyFileExists(htmlDir, "searchindex.js")
def test_failToBuild(self):
"""
Check that SphinxBuilder.build fails when run against a non-sphinx
directory.
"""
# note no fake sphinx project is created
self.assertRaises(CalledProcessError, self.builder.build, self.sphinxDir)
#!/usr/bin/env python
"""
Twisted moved the C{twisted} hierarchy to the C{src} hierarchy, but C{git}
doesn't know how to track moves of directories, only files. Therefore any
files added in branches after this move will be added into ./twisted/ and need
to be moved over into
"""
import os
from twisted.python.filepath import FilePath
here = FilePath(__file__).parent().parent()
fromPath = here.child("twisted")
toPath = here.child("src")
for fn in fromPath.walk():
if fn.isfile():
os.system(
"git mv {it} src/{it}".format(it="/".join(fn.segmentsFrom(here))))
os.system('git clean -fd')
def test_build(self):
"""
L{APIBuilder.build} writes an index file which includes the name of the
project specified.
"""
stdout = BytesIO()
self.patch(sys, "stdout", stdout)
projectName = "Foobar"
packageName = "quux"
projectURL = "scheme:project"
sourceURL = "scheme:source"
docstring = "text in docstring"
privateDocstring = "should also appear in output"
inputPath = FilePath(self.mktemp()).child(packageName)
inputPath.makedirs()
inputPath.child("__init__.py").setContent(
"def foo():\n"
" '{}'\n"
"def _bar():\n"
" '{}'".format(docstring, privateDocstring).encode()
)
outputPath = FilePath(self.mktemp())
builder = APIBuilder()
builder.build(projectName, projectURL, sourceURL, inputPath, outputPath)
indexPath = outputPath.child("index.html")
self.assertTrue(
indexPath.exists(), f"API index {outputPath.path!r} did not exist."
)
self.assertIn(
f'<a href="{projectURL}">{projectName}</a>',
indexPath.getContent().decode(),
"Project name/location not in file contents.",
)
quuxPath = outputPath.child("quux.html")
self.assertTrue(
quuxPath.exists(),
f"Package documentation file {quuxPath.path!r} did not exist.",
)
self.assertIn(
docstring,
quuxPath.getContent().decode(),
"Docstring not in package documentation file.",
)
self.assertIn(
f'<a href="{sourceURL}/{packageName}/__init__.py">(source)</a>',
quuxPath.getContent().decode(),
)
self.assertIn(
'<a class="functionSourceLink" href="%s/%s/__init__.py#L1">'
% (sourceURL, packageName),
quuxPath.getContent().decode(),
)
self.assertIn(privateDocstring, quuxPath.getContent().decode())
self.assertEqual(stdout.getvalue(), b"")
from twisted.application.service import Application
from twisted.application.internet import (TCPServer, SSLServer)
from twisted.python.filepath import FilePath
from twisted.web.server import Site
from twisted.web.static import File
from twisted.web.resource import Resource
from twisted.web._responses import NOT_FOUND
from twisted.web.util import redirectTo
import pem
# the actual html/css/js... files being served are all located here
DOCUMENTS = FilePath(__file__).parent().child("public")
ERROR_RESOURCE = DOCUMENTS.child("error").child("index.html")
INSECURE_PORT = os.environ.get("INSECURE_PORT")
SECURE_PORT = os.environ.get("SECURE_PORT")
# all certificates are passed in as environment variables
PRIVATE_KEY = os.environ.get("PRIVATE_KEY")
CERTIFICATE_CHAIN = os.environ.get("CERTIFICATE_CHAIN")
DH_PARAMETER = os.environ.get("DH_PARAMETER")
ctxFactory = pem.certificateOptionsFromPEMs(
pem.parse(PRIVATE_KEY),
pem.parse(CERTIFICATE_CHAIN),
dhParameters=pem.DiffieHellmanParameters(DH_PARAMETER))
""" Crochet tests. """ from twisted.python.filepath import FilePath # Directory where the crochet package is stored: crochet_directory = FilePath(__file__).parent().parent().parent().path
class BuildScriptsTests(TestCase):
"""
Tests for L{dist.build_scripts_twisted}.
"""
def setUp(self):
self.source = FilePath(self.mktemp())
self.target = FilePath(self.mktemp())
self.source.makedirs()
self.addCleanup(os.chdir, os.getcwd())
os.chdir(self.source.path)
def buildScripts(self):
"""
Write 3 types of scripts and run the L{build_scripts_twisted}
command.
"""
self.writeScript(self.source, "script1",
("#! /usr/bin/env python2.7\n"
"# bogus script w/ Python sh-bang\n"
"pass\n"))
self.writeScript(self.source, "script2.py",
("#!/usr/bin/python\n"
"# bogus script w/ Python sh-bang\n"
"pass\n"))
self.writeScript(self.source, "shell.sh",
("#!/bin/sh\n"
"# bogus shell script w/ sh-bang\n"
"exit 0\n"))
expected = ['script1', 'script2.py', 'shell.sh']
cmd = self.getBuildScriptsCmd(
self.target, [self.source.child(fn).path for fn in expected])
cmd.finalize_options()
cmd.run()
return self.target.listdir()
def getBuildScriptsCmd(self, target, scripts):
"""
Create a distutils L{Distribution} with a L{DummyCommand} and wrap it
in L{build_scripts_twisted}.
@type target: L{FilePath}
"""
dist = Distribution()
dist.scripts = scripts
dist.command_obj["build"] = DummyCommand(build_scripts=target.path,
force=1,
executable=sys.executable)
return build_scripts_twisted(dist)
def writeScript(self, dir, name, text):
"""
Write the script to disk.
"""
with open(dir.child(name).path, "w") as f:
f.write(text)
def test_notWindows(self):
"""
L{build_scripts_twisted} does not rename scripts on non-Windows
platforms.
"""
self.patch(os, "name", "twisted")
built = self.buildScripts()
for name in ['script1', 'script2.py', 'shell.sh']:
self.assertIn(name, built)
def test_windows(self):
"""
L{build_scripts_twisted} renames scripts so they end with '.py' on
the Windows platform.
"""
self.patch(os, "name", "nt")
built = self.buildScripts()
for name in ['script1.py', 'script2.py', 'shell.sh.py']:
self.assertIn(name, built)
#!/usr/bin/env python
"""
A performance benchmark using the example from issue #232:
https://github.com/Julian/jsonschema/pull/232
"""
import jsonschema
from jsonschema.tests._suite import Collection
from perf import Runner
from pyrsistent import m
from twisted.python.filepath import FilePath
collection = Collection(
path=FilePath(__file__).sibling("issue232"),
remotes=m(),
name="issue232",
validator=jsonschema.Draft7Validator,
)
if __name__ == "__main__":
collection.benchmark(runner=Runner())
from twisted.web import server, client, error, resource
from twisted.web.static import Data
from twisted.web.util import Redirect
from twisted.internet import reactor, defer, interfaces
from twisted.python.filepath import FilePath
from twisted.python.log import msg
from twisted.protocols.policies import WrappingFactory
from twisted.test.proto_helpers import StringTransport
try:
from twisted.internet import ssl
except:
ssl = None
from twisted import test
serverPEM = FilePath(test.__file__).sibling('server.pem')
serverPEMPath = serverPEM.asBytesMode().path
class ExtendedRedirect(resource.Resource):
"""
Redirection resource.
The HTTP status code is set according to the C{code} query parameter.
@type lastMethod: C{bytes}
@ivar lastMethod: Last handled HTTP request method
"""
isLeaf = True
lastMethod = None
def setUp(self):
self.source = FilePath(self.mktemp())
self.target = FilePath(self.mktemp())
self.source.makedirs()
self.addCleanup(os.chdir, os.getcwd())
os.chdir(self.source.path)
class WriteSessions(unittest.TestCase):
port = 65466
def setUp(self):
self.clock = Clock()
self.tmp_dir_path = tempfile.mkdtemp()
self.target = FilePath(self.tmp_dir_path).child('foo')
self.writer = DelayedWriter(self.target, _clock=self.clock, delay=2)
self.transport = FakeTransport(hostAddress=('127.0.0.1', self.port))
self.ws = WriteSession(self.writer, _clock=self.clock)
self.ws.timeout = (4, 4, 4)
self.ws.transport = self.transport
self.ws.startProtocol()
def test_ERROR(self):
err_dgram = ERRORDatagram.from_code(ERR_NOT_DEFINED, 'no reason')
self.ws.datagramReceived(err_dgram)
self.clock.advance(0.1)
self.failIf(self.transport.value())
self.failUnless(self.transport.disconnecting)
@inlineCallbacks
def test_DATA_stale_blocknum(self):
self.ws.block_size = 6
self.ws.blocknum = 2
data_datagram = DATADatagram(1, 'foobar')
yield self.ws.datagramReceived(data_datagram)
self.writer.finish()
self.failIf(self.target.open('r').read())
self.failIf(self.transport.disconnecting)
ack_dgram = TFTPDatagramFactory(*split_opcode(self.transport.value()))
self.assertEqual(ack_dgram.blocknum, 1)
self.addCleanup(self.ws.cancel)
@inlineCallbacks
def test_DATA_invalid_blocknum(self):
self.ws.block_size = 6
data_datagram = DATADatagram(3, 'foobar')
yield self.ws.datagramReceived(data_datagram)
self.writer.finish()
self.failIf(self.target.open('r').read())
self.failIf(self.transport.disconnecting)
err_dgram = TFTPDatagramFactory(*split_opcode(self.transport.value()))
self.assert_(isinstance(err_dgram, ERRORDatagram))
self.addCleanup(self.ws.cancel)
def test_DATA(self):
self.ws.block_size = 6
data_datagram = DATADatagram(1, 'foobar')
d = self.ws.datagramReceived(data_datagram)
def cb(ign):
self.clock.advance(0.1)
#self.writer.finish()
#self.assertEqual(self.target.open('r').read(), 'foobar')
self.failIf(self.transport.disconnecting)
ack_dgram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.assertEqual(ack_dgram.blocknum, 1)
self.failIf(
self.ws.completed,
"Data length is equal to blocksize, no reason to stop")
data_datagram = DATADatagram(2, 'barbaz')
self.transport.clear()
d = self.ws.datagramReceived(data_datagram)
d.addCallback(cb_)
self.clock.advance(3)
return d
def cb_(ign):
self.clock.advance(0.1)
self.failIf(self.transport.disconnecting)
ack_dgram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.assertEqual(ack_dgram.blocknum, 2)
self.failIf(
self.ws.completed,
"Data length is equal to blocksize, no reason to stop")
d.addCallback(cb)
self.addCleanup(self.ws.cancel)
self.clock.advance(3)
return d
def test_DATA_finished(self):
self.ws.block_size = 6
# Send a terminating datagram
data_datagram = DATADatagram(1, 'foo')
d = self.ws.datagramReceived(data_datagram)
def cb(res):
self.clock.advance(0.1)
self.assertEqual(self.target.open('r').read(), 'foo')
ack_dgram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.failUnless(isinstance(ack_dgram, ACKDatagram))
self.failUnless(
self.ws.completed,
"Data length is less, than blocksize, time to stop")
self.transport.clear()
# Send another datagram after the transfer is considered complete
data_datagram = DATADatagram(2, 'foobar')
self.ws.datagramReceived(data_datagram)
self.assertEqual(self.target.open('r').read(), 'foo')
err_dgram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.failUnless(isinstance(err_dgram, ERRORDatagram))
# Check for proper disconnection after grace timeout expires
self.clock.pump((4, ) * 4)
self.failUnless(
self.transport.disconnecting,
"We are done and the grace timeout is over, should disconnect")
d.addCallback(cb)
self.clock.advance(2)
return d
def test_DATA_backoff(self):
self.ws.block_size = 5
data_datagram = DATADatagram(1, 'foobar')
d = self.ws.datagramReceived(data_datagram)
def cb(ign):
self.clock.advance(0.1)
ack_datagram = ACKDatagram(1)
self.clock.pump((1, ) * 5)
# Sent two times - initial send and a retransmit after first timeout
self.assertEqual(self.transport.value(),
ack_datagram.to_wire() * 2)
# Sent three times - initial send and two retransmits
self.clock.pump((1, ) * 4)
self.assertEqual(self.transport.value(),
ack_datagram.to_wire() * 3)
# Sent still three times - initial send, two retransmits and the last wait
self.clock.pump((1, ) * 4)
self.assertEqual(self.transport.value(),
ack_datagram.to_wire() * 3)
self.failUnless(self.transport.disconnecting)
d.addCallback(cb)
self.clock.advance(2.1)
return d
@inlineCallbacks
def test_failed_write(self):
self.writer.cancel()
self.ws.writer = FailingWriter()
data_datagram = DATADatagram(1, 'foobar')
yield self.ws.datagramReceived(data_datagram)
self.flushLoggedErrors()
self.clock.advance(0.1)
err_datagram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.failUnless(isinstance(err_datagram, ERRORDatagram))
self.failUnless(self.transport.disconnecting)
def test_time_out(self):
data_datagram = DATADatagram(1, 'foobar')
d = self.ws.datagramReceived(data_datagram)
def cb(ign):
self.clock.pump((1, ) * 13)
self.failUnless(self.transport.disconnecting)
d.addCallback(cb)
self.clock.advance(4)
return d
def tearDown(self):
shutil.rmtree(self.tmp_dir_path)
def cbFinished(ignored):
self.assertEqual(
FilePath(output).getContent(),
b"[('foo', 'bar')]")
class SSHUserAuthClientTests(TestCase):
"""
Tests for L{SSHUserAuthClient}.
@type rsaPublic: L{Key}
@ivar rsaPublic: A public RSA key.
"""
def setUp(self):
self.rsaPublic = Key.fromString(keydata.publicRSA_openssh)
self.tmpdir = FilePath(self.mktemp())
self.tmpdir.makedirs()
self.rsaFile = self.tmpdir.child('id_rsa')
self.rsaFile.setContent(keydata.privateRSA_openssh)
self.tmpdir.child('id_rsa.pub').setContent(keydata.publicRSA_openssh)
def test_signDataWithAgent(self):
"""
When connected to an agent, L{SSHUserAuthClient} can use it to
request signatures of particular data with a particular L{Key}.
"""
client = SSHUserAuthClient("user", ConchOptions(), None)
agent = SSHAgentClient()
transport = StringTransport()
agent.makeConnection(transport)
client.keyAgent = agent
cleartext = "Sign here"
client.signData(self.rsaPublic, cleartext)
self.assertEqual(
transport.value(),
"\x00\x00\x00\x8b\r\x00\x00\x00u" + self.rsaPublic.blob() +
"\x00\x00\x00\t" + cleartext +
"\x00\x00\x00\x00")
def test_agentGetPublicKey(self):
"""
L{SSHUserAuthClient} looks up public keys from the agent using the
L{SSHAgentClient} class. That L{SSHAgentClient.getPublicKey} returns a
L{Key} object with one of the public keys in the agent. If no more
keys are present, it returns L{None}.
"""
agent = SSHAgentClient()
agent.blobs = [self.rsaPublic.blob()]
key = agent.getPublicKey()
self.assertTrue(key.isPublic())
self.assertEqual(key, self.rsaPublic)
self.assertIsNone(agent.getPublicKey())
def test_getPublicKeyFromFile(self):
"""
L{SSHUserAuthClient.getPublicKey()} is able to get a public key from
the first file described by its options' C{identitys} list, and return
the corresponding public L{Key} object.
"""
options = ConchOptions()
options.identitys = [self.rsaFile.path]
client = SSHUserAuthClient("user", options, None)
key = client.getPublicKey()
self.assertTrue(key.isPublic())
self.assertEqual(key, self.rsaPublic)
def test_getPublicKeyAgentFallback(self):
"""
If an agent is present, but doesn't return a key,
L{SSHUserAuthClient.getPublicKey} continue with the normal key lookup.
"""
options = ConchOptions()
options.identitys = [self.rsaFile.path]
agent = SSHAgentClient()
client = SSHUserAuthClient("user", options, None)
client.keyAgent = agent
key = client.getPublicKey()
self.assertTrue(key.isPublic())
self.assertEqual(key, self.rsaPublic)
def test_getPublicKeyBadKeyError(self):
"""
If L{keys.Key.fromFile} raises a L{keys.BadKeyError}, the
L{SSHUserAuthClient.getPublicKey} tries again to get a public key by
calling itself recursively.
"""
options = ConchOptions()
self.tmpdir.child('id_dsa.pub').setContent(keydata.publicDSA_openssh)
dsaFile = self.tmpdir.child('id_dsa')
dsaFile.setContent(keydata.privateDSA_openssh)
options.identitys = [self.rsaFile.path, dsaFile.path]
self.tmpdir.child('id_rsa.pub').setContent('not a key!')
client = SSHUserAuthClient("user", options, None)
key = client.getPublicKey()
self.assertTrue(key.isPublic())
self.assertEqual(key, Key.fromString(keydata.publicDSA_openssh))
self.assertEqual(client.usedFiles, [self.rsaFile.path, dsaFile.path])
def test_getPrivateKey(self):
"""
L{SSHUserAuthClient.getPrivateKey} will load a private key from the
last used file populated by L{SSHUserAuthClient.getPublicKey}, and
return a L{Deferred} which fires with the corresponding private L{Key}.
"""
rsaPrivate = Key.fromString(keydata.privateRSA_openssh)
options = ConchOptions()
options.identitys = [self.rsaFile.path]
client = SSHUserAuthClient("user", options, None)
# Populate the list of used files
client.getPublicKey()
def _cbGetPrivateKey(key):
self.assertFalse(key.isPublic())
self.assertEqual(key, rsaPrivate)
return client.getPrivateKey().addCallback(_cbGetPrivateKey)
def test_getPrivateKeyPassphrase(self):
"""
L{SSHUserAuthClient} can get a private key from a file, and return a
Deferred called back with a private L{Key} object, even if the key is
encrypted.
"""
rsaPrivate = Key.fromString(keydata.privateRSA_openssh)
passphrase = 'this is the passphrase'
self.rsaFile.setContent(rsaPrivate.toString('openssh', passphrase))
options = ConchOptions()
options.identitys = [self.rsaFile.path]
client = SSHUserAuthClient("user", options, None)
# Populate the list of used files
client.getPublicKey()
def _getPassword(prompt):
self.assertEqual(prompt,
"Enter passphrase for key '%s': " % (
self.rsaFile.path,))
return passphrase
def _cbGetPrivateKey(key):
self.assertFalse(key.isPublic())
self.assertEqual(key, rsaPrivate)
self.patch(client, '_getPassword', _getPassword)
return client.getPrivateKey().addCallback(_cbGetPrivateKey)
def stubFinder():
return FilePath(__file__).sibling('segfault.py').path
from persona import CamCheck
from twisted.python.filepath import FilePath
from twisted.python import failure
from identitytoolkit import gitkitclient
import gi
gi.require_version('Gst','1.0')
from gi.repository import GObject, Gst
GObject.threads_init()
Gst.init(None)
from autobahn.twisted.websocket import WebSocketServerFactory, \
WebSocketServerProtocol
from autobahn.twisted.resource import WebSocketResource
p12_file = FilePath('/home/chetan/pscore/protected/PacketServo-267cefd0c2d6.p12')
f = p12_file.open('r')
key = f.read()
f.close()
gitkit_instance = gitkitclient.GitkitClient(
#gitkit_instance = gitkitclient.FromConfigFile(config_json)(
client_id="912263063433-i8se3d23v29chlnocovc4umi8cuqdbmd.apps.googleusercontent.com",
service_account_email="*****@*****.**",
service_account_key=key,
widget_url="http://www.packetservo.com/auth/login"
)
class WebMStream(object):
def __init__(self):
self.make_pipeline()
class ReadSessions(unittest.TestCase):
test_data = """line1
line2
anotherline"""
port = 65466
def setUp(self):
self.clock = Clock()
self.tmp_dir_path = tempfile.mkdtemp()
self.target = FilePath(self.tmp_dir_path).child('foo')
with self.target.open('wb') as temp_fd:
temp_fd.write(self.test_data)
self.reader = DelayedReader(self.target, _clock=self.clock, delay=2)
self.transport = FakeTransport(hostAddress=('127.0.0.1', self.port))
self.rs = ReadSession(self.reader, _clock=self.clock)
self.rs.transport = self.transport
self.rs.startProtocol()
@inlineCallbacks
def test_ERROR(self):
err_dgram = ERRORDatagram.from_code(ERR_NOT_DEFINED, 'no reason')
yield self.rs.datagramReceived(err_dgram)
self.failIf(self.transport.value())
self.failUnless(self.transport.disconnecting)
@inlineCallbacks
def test_ACK_invalid_blocknum(self):
ack_datagram = ACKDatagram(3)
yield self.rs.datagramReceived(ack_datagram)
self.failIf(self.transport.disconnecting)
err_dgram = TFTPDatagramFactory(*split_opcode(self.transport.value()))
self.assert_(isinstance(err_dgram, ERRORDatagram))
self.addCleanup(self.rs.cancel)
@inlineCallbacks
def test_ACK_stale_blocknum(self):
self.rs.blocknum = 2
ack_datagram = ACKDatagram(1)
yield self.rs.datagramReceived(ack_datagram)
self.failIf(self.transport.disconnecting)
self.failIf(self.transport.value(),
"Stale ACK datagram, we should not write anything back")
self.addCleanup(self.rs.cancel)
def test_ACK(self):
self.rs.block_size = 5
self.rs.blocknum = 1
ack_datagram = ACKDatagram(1)
d = self.rs.datagramReceived(ack_datagram)
def cb(ign):
self.clock.advance(0.1)
self.failIf(self.transport.disconnecting)
data_datagram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.assertEqual(data_datagram.data, 'line1')
self.failIf(
self.rs.completed,
"Got enough bytes from the reader, there is no reason to stop")
d.addCallback(cb)
self.clock.advance(2.5)
self.addCleanup(self.rs.cancel)
return d
def test_ACK_finished(self):
self.rs.block_size = 512
self.rs.blocknum = 1
# Send a terminating datagram
ack_datagram = ACKDatagram(1)
d = self.rs.datagramReceived(ack_datagram)
def cb(ign):
self.clock.advance(0.1)
ack_datagram = ACKDatagram(2)
# This datagram doesn't trigger any sends
self.rs.datagramReceived(ack_datagram)
self.assertEqual(self.transport.value(),
DATADatagram(2, self.test_data).to_wire())
self.failUnless(
self.rs.completed,
"Data length is less, than blocksize, time to stop")
self.addCleanup(self.rs.cancel)
d.addCallback(cb)
self.clock.advance(3)
return d
def test_ACK_backoff(self):
self.rs.block_size = 5
self.rs.blocknum = 1
ack_datagram = ACKDatagram(1)
d = self.rs.datagramReceived(ack_datagram)
def cb(ign):
self.clock.pump((1, ) * 4)
# Sent two times - initial send and a retransmit after first timeout
self.assertEqual(self.transport.value(),
DATADatagram(2, self.test_data[:5]).to_wire() * 2)
# Sent three times - initial send and two retransmits
self.clock.pump((1, ) * 5)
self.assertEqual(self.transport.value(),
DATADatagram(2, self.test_data[:5]).to_wire() * 3)
# Sent still three times - initial send, two retransmits and the last wait
self.clock.pump((1, ) * 10)
self.assertEqual(self.transport.value(),
DATADatagram(2, self.test_data[:5]).to_wire() * 3)
self.failUnless(self.transport.disconnecting)
d.addCallback(cb)
self.clock.advance(2.5)
return d
@inlineCallbacks
def test_failed_read(self):
self.reader.finish()
self.rs.reader = FailingReader()
self.rs.blocknum = 1
ack_datagram = ACKDatagram(1)
yield self.rs.datagramReceived(ack_datagram)
self.flushLoggedErrors()
self.clock.advance(0.1)
err_datagram = TFTPDatagramFactory(
*split_opcode(self.transport.value()))
self.failUnless(isinstance(err_datagram, ERRORDatagram))
self.failUnless(self.transport.disconnecting)
def tearDown(self):
shutil.rmtree(self.tmp_dir_path)
def test_basicOperation(self):
"""
Running the L{tap2deb} script produces a bunch of files using
C{dpkg-buildpackage}.
"""
# Skip tests if dpkg-buildpackage is not present
if not procutils.which("dpkg-buildpackage"):
raise SkipTest("dpkg-buildpackage must be present to test tap2deb")
baseDir = FilePath(self.mktemp())
baseDir.makedirs()
# Make a temporary .tap file
version = '1.0'
tapName = 'lemon'
tapFile = baseDir.child("%s.tap" % (tapName, ))
tapFile.setContent("# Dummy .tap file")
buildDir = FilePath('.build')
outputDir = buildDir.child('twisted-%s-%s' % (tapName, version))
# Run
args = ["--tapfile", tapFile.path, "--maintainer", self.maintainer]
tap2deb.run(args)
# Verify input files were created
self.assertEqual(
sorted(outputDir.listdir()),
['build-stamp', 'debian', 'install-stamp', 'lemon.tap'])
debianDir = outputDir.child('debian')
for name in [
'README.Debian', 'conffiles', 'default', 'init.d', 'postinst',
'prerm', 'postrm', 'changelog', 'control', 'copyright', 'dirs',
'rules'
]:
self.assertTrue(debianDir.child(name).exists())
# Verify 4 output files were created
self.assertTrue(buildDir.child('twisted-lemon_1.0_all.deb').exists())
self.assertTrue(buildDir.child('twisted-lemon_1.0.tar.gz').exists())
self.assertTrue(buildDir.child('twisted-lemon_1.0.dsc').exists())
self.assertEqual(
len(buildDir.globChildren('twisted-lemon_1.0_*.changes')), 1)
from characteristic import attributes
from twisted.internet.defer import maybeDeferred
from twisted.internet.task import deferLater
from twisted.python.filepath import FilePath
from twisted.application.service import Service
from twisted.internet.defer import fail
# We might want to make these utilities shared, rather than in zfs
# module... but in this case the usage is temporary and should go away as
# part of https://github.com/ClusterHQ/flocker/issues/64
from .filesystems.zfs import StoragePool
from ._model import VolumeSize
from ..common.script import ICommandLineScript
DEFAULT_CONFIG_PATH = FilePath(b"/etc/flocker/volume.json")
FLOCKER_MOUNTPOINT = FilePath(b"/flocker")
FLOCKER_POOL = b"flocker"
WAIT_FOR_VOLUME_INTERVAL = 0.1
class CreateConfigurationError(Exception):
"""Create the configuration file failed."""
@attributes(["namespace", "id"])
class VolumeName(object):
"""
The volume and its copies' name within the cluster.
# See LICENSE for details.
"""
Helper classes for twisted.test.test_ssl.
They are in a separate module so they will not prevent test_ssl importing if
pyOpenSSL is unavailable.
"""
from OpenSSL import SSL
from twisted.internet import ssl
from twisted.python.compat import nativeString
from twisted.python.filepath import FilePath
certPath = nativeString(
FilePath(__file__.encode("utf-8")).sibling(b"server.pem").path)
class ClientTLSContext(ssl.ClientContextFactory):
"""
SSL Context Factory for client-side connections.
"""
isClient = 1
def getContext(self):
"""
Return an L{SSL.Context} to be use for client-side connections.
Will not return a cached context.
This is done to improve the test coverage as most implementation
def main():
plugins_dir = FilePath("/run/docker/plugins/")
if not plugins_dir.exists():
plugins_dir.makedirs()
dvol_path = FilePath("/var/lib/dvol/volumes")
if not dvol_path.exists():
dvol_path.makedirs()
voluminous = Voluminous(dvol_path.path)
sock = plugins_dir.child("%s.sock" % (VOLUME_DRIVER_NAME, ))
if sock.exists():
sock.remove()
adapterServer = internet.UNIXServer(sock.path, getAdapter(voluminous))
reactor.callWhenRunning(adapterServer.startService)
reactor.run()
def run(self):
"""Override Process run method to provide a custom wrapper for the API.
This provides a continuous loop for watching the service while keeping
an ear open to the main process from openContentPlatform, listening for
any interrupt requests.
"""
## Setup requested log handler
try:
## Twisted imports here to avoid issues with epoll on Linux
from twisted.internet import reactor, ssl
from twisted.python.filepath import FilePath
from twisted.web.server import Site
from twisted.web.wsgi import WSGIResource
from twisted.python.threadpool import ThreadPool
print('Starting {}'.format(self.serviceName))
self.getLocalLogger()
self.logger.info('Starting {}'.format(self.serviceName))
self.logger.info('Setting up service communication...')
## Setup shared resources for our WSGIResource instances to use
self.getSharedLogger()
self.getSharedDbPool()
## Create a PID file for system administration purposes
pidEntryService(self.serviceName, env, self.pid)
## Reference the magic WSGI throwable from the module using Hug
application = serviceCommunication.__hug_wsgi__
## Setup the WSGI to be hosted through Twisted's web server
wsgiThreadPool = ThreadPool()
wsgiThreadPool.start()
## For some reason the system event wasn't working all the time,
## so I'm adding an explicit wsgiThreadPool.stop() below as well,
## which was needed before reactor.stop() would properly cleanup.
reactor.addSystemEventTrigger('after', 'shutdown',
wsgiThreadPool.stop)
resource = WSGIResource(reactor, wsgiThreadPool, application)
self.logger.info('calling listener on {}:{}.'.format(
str(self.serviceEndpoint), self.listeningPort))
if self.useCertificates:
## Use TLS to encrypt the communication
certData = FilePath(
os.path.join(
env.privateInternalCertPath,
self.globalSettings.get(
'ocpCertificateCaFile'))).getContent()
certificate = ssl.PrivateCertificate.loadPEM(certData)
reactor.listenSSL(self.listeningPort, Site(resource),
certificate.options())
else:
## Plain text communication
reactor.listenTCP(self.listeningPort,
Site(resource),
interface=self.serviceEndpoint)
## Normally we'd just call reactor.run() here and let twisted handle
## the wait loop while watching for signals. The problem is that we
## need openContentPlatform (parent process) to manage this process.
## So this is a bit hacky in that I'm using the reactor code, but I
## am manually calling what would be called if I just called run():
reactor.startRunning()
## Start event wait loop
while reactor._started and not self.shutdownEvent.is_set():
try:
## Four lines from twisted.internet.main.mainloop:
reactor.runUntilCurrent()
t2 = reactor.timeout()
t = reactor.running and t2
reactor.doIteration(t)
except:
exception = traceback.format_exception(
sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
self.logger.error('Exception in {}: {}'.format(
self.serviceName, str(exception)))
break
if self.shutdownEvent.is_set():
self.logger.info('Process received shutdownEvent')
with suppress(Exception):
wsgiThreadPool.stop()
with suppress(Exception):
reactor.stop()
except:
exception = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
self.logger.error('Exception in {}: {}'.format(
self.serviceName, str(exception)))
## Cleanup
pidRemoveService(self.serviceName, env, self.pid)
self.logger.info('Stopped {}'.format(self.serviceName))
print('Stopped {}'.format(self.serviceName))
## end run
return
class WebUIAPI(object):
app = Klein()
# Maximum number in seconds after which to return a result even if no
# change happened.
_long_polling_timeout = 30
_reactor = reactor
_enable_xsrf_protection = True
def __init__(self, config, director, scheduler, _reactor=reactor):
self._reactor = reactor
self.director = director
self.scheduler = scheduler
self.config = config
self.measurement_path = FilePath(config.measurements_directory)
# We use a double submit token to protect against XSRF
rng = SystemRandom()
token_space = string.letters + string.digits
self._xsrf_token = b''.join(
[rng.choice(token_space) for _ in range(30)])
self._director_started = False
self._is_initialized = config.is_initialized()
# We use exponential backoff to trigger retries of the startup of
# the director.
self._director_startup_retries = 0
# Maximum delay should be 30 minutes
self._director_max_retry_delay = 30 * 60
self.status_poller = LongPoller(self._long_polling_timeout, _reactor)
self.director_event_poller = LongPoller(self._long_polling_timeout,
_reactor)
# XXX move this elsewhere
self.director_event_poller.start()
self.status_poller.start()
self.director.subscribe(self.handle_director_event)
if self._is_initialized:
self.start_director()
def start_director(self):
log.debug("Starting director")
d = self.director.start()
d.addCallback(self.director_started)
d.addErrback(self.director_startup_failed)
d.addBoth(lambda _: self.status_poller.notify())
@property
def status(self):
quota_warning = None
try:
with open(os.path.join(config.running_path,
"quota_warning")) as in_file:
quota_warning = in_file.read()
except IOError as ioe:
if ioe.errno != errno.ENOENT:
raise
return {
"software_version": ooniprobe_version,
"software_name": "ooniprobe",
"asn": probe_ip.geodata['asn'],
"country_code": probe_ip.geodata['countrycode'],
"director_started": self._director_started,
"initialized": self._is_initialized,
"quota_warning": quota_warning
}
def handle_director_event(self, event):
log.debug("Handling event {0}".format(event.type))
self.director_event_poller.notify(event)
def director_startup_failed(self, failure):
self._director_startup_retries += 1
# We delay the startup using binary exponential backoff with an
# upper bound.
startup_delay = random.uniform(
0,
min(2**self._director_startup_retries,
self._director_max_retry_delay))
log.err("Failed to start the director, "
"retrying in {0}s".format(startup_delay))
self._reactor.callLater(startup_delay, self.start_director)
def director_started(self, _):
log.debug("Started director")
self._director_started = True
@app.handle_errors(NotFound)
@xsrf_protect(check=False)
def not_found(self, request, _):
request.redirect('/client/')
@app.handle_errors(WebUIError)
@xsrf_protect(check=False)
def web_ui_error(self, request, failure):
error = failure.value
request.setResponseCode(error.code)
return self.render_json(
{
"error_code": error.code,
"error_message": error.message
}, request)
def render_json(self, obj, request):
json_string = json.dumps(obj) + "\n"
request.setHeader('Content-Type', 'application/json')
request.setHeader('Content-Length', len(json_string))
return json_string
@app.route('/api/notify', methods=["GET"])
@xsrf_protect(check=False)
def api_notify(self, request):
def got_director_event(event):
return self.render_json(
{
"type": event.type,
"message": event.message
}, request)
d = self.director_event_poller.get()
d.addCallback(got_director_event)
return d
@app.route('/api/status', methods=["GET"])
@xsrf_protect(check=False)
def api_status(self, request):
return self.render_json(self.status, request)
@app.route('/api/status/update', methods=["GET"])
@xsrf_protect(check=False)
def api_status_update(self, request):
def got_status_update(event):
return self.api_status(request)
d = self.status_poller.get()
d.addCallback(got_status_update)
return d
@app.route('/api/initialize', methods=["GET"])
@xsrf_protect(check=False)
@requires_false(attrs=['_is_initialized'])
def api_initialize_get(self, request):
available_decks = []
for deck_id, deck in self.director.deck_store.list():
available_decks.append({
'name':
deck.name,
'description':
deck.description,
'schedule':
deck.schedule,
'enabled':
self.director.deck_store.is_enabled(deck_id),
'id':
deck_id,
'icon':
deck.icon
})
return self.render_json({"available_decks": available_decks}, request)
@app.route('/api/initialize', methods=["POST"])
@xsrf_protect(check=True)
@requires_false(attrs=['_is_initialized'])
def api_initialize(self, request):
try:
initial_configuration = json.load(request.content)
except ValueError:
raise WebUIError(400, 'Invalid JSON message recevied')
required_keys = [
'include_ip', 'include_asn', 'include_country', 'should_upload',
'preferred_backend'
]
options = {}
for required_key in required_keys:
try:
options[required_key] = initial_configuration[required_key]
except KeyError:
raise WebUIError(
400, 'Missing required key {0}'.format(required_key))
config.create_config_file(**options)
try:
deck_config = initial_configuration['deck_config']
except KeyError:
raise WebUIError(400, 'Missing enabled decks')
for deck_id, enabled in deck_config.items():
try:
if enabled is True:
self.director.deck_store.enable(deck_id)
elif enabled is False:
try:
self.director.deck_store.disable(deck_id)
except DeckNotFound:
# We ignore these errors, because it could be that a deck
# that is marked as disabled is already disabled
pass
except DeckNotFound:
raise WebUIError(404, 'Deck not found')
config.set_initialized()
self.scheduler.refresh_deck_list()
self._is_initialized = True
self.status_poller.notify()
self.start_director()
return self.render_json({"result": "ok"}, request)
@app.route('/api/deck/<string:deck_id>/start', methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_start(self, request, deck_id):
try:
deck = self.director.deck_store.get(deck_id)
except DeckNotFound:
raise WebUIError(404, "Deck not found")
try:
self.run_deck(deck)
except:
raise WebUIError(500, "Failed to start deck")
return self.render_json({"status": "started " + deck.name}, request)
@app.route('/api/deck', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_deck_list(self, request):
deck_list = {'decks': []}
for deck_id, deck in self.director.deck_store.list():
nettests = []
for task in deck.tasks:
if task.type == 'ooni':
assert task.ooni['test_name'] is not None
nettests.append(task.ooni['test_name'])
deck_list['decks'].append({
'id':
deck_id,
'name':
deck.name,
'icon':
deck.icon,
'running':
self.director.isDeckRunning(deck_id, from_schedule=False),
'running_scheduled':
self.director.isDeckRunning(deck_id, from_schedule=True),
'nettests':
nettests,
'description':
deck.description,
'schedule':
deck.schedule,
'enabled':
self.director.deck_store.is_enabled(deck_id)
})
return self.render_json(deck_list, request)
@app.route('/api/deck/<string:deck_id>/run', methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_run(self, request, deck_id):
try:
deck = self.director.deck_store.get(deck_id)
except DeckNotFound:
raise WebUIError(404, "Deck not found")
self.run_deck(deck)
return self.render_json({"status": "starting"}, request)
@app.route('/api/deck/<string:deck_id>/enable', methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_enable(self, request, deck_id):
try:
self.director.deck_store.enable(deck_id)
except DeckNotFound:
raise WebUIError(404, "Deck not found")
self.scheduler.refresh_deck_list()
return self.render_json({"status": "enabled"}, request)
@app.route('/api/deck/<string:deck_id>/disable', methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_director_started', '_is_initialized'])
def api_deck_disable(self, request, deck_id):
try:
self.director.deck_store.disable(deck_id)
except DeckNotFound:
raise WebUIError(404, "Deck not found")
self.scheduler.refresh_deck_list()
return self.render_json({"status": "disabled"}, request)
@defer.inlineCallbacks
def run_deck(self, deck):
# These are dangling deferreds
try:
yield deck.setup()
yield deck.run(self.director, from_schedule=False)
self.director_event_poller.notify(
DirectorEvent("success", "Started Deck " + deck.id))
except:
self.director_event_poller.notify(
DirectorEvent("error", "Failed to start deck"))
@app.route('/api/nettest/<string:test_name>/start', methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_director_started', '_is_initialized'])
def api_nettest_start(self, request, test_name):
try:
_ = self.director.netTests[test_name]
except KeyError:
raise WebUIError(500, 'Could not find the specified test')
try:
test_options = json.load(request.content)
except ValueError:
raise WebUIError(500, 'Invalid JSON message recevied')
test_options["test_name"] = test_name
deck_data = {"tasks": [{"ooni": test_options}]}
try:
deck = NGDeck()
deck.load(deck_data)
self.run_deck(deck)
except errors.MissingRequiredOption as option_name:
raise WebUIError(
400, 'Missing required option: "{}"'.format(option_name))
except usage.UsageError as ue:
raise WebUIError(400, 'Error in parsing options')
except errors.InsufficientPrivileges:
raise WebUIError(400, 'Insufficient privileges')
except Exception as exc:
log.exception(exc)
raise WebUIError(500, 'Failed to start nettest')
return self.render_json({"status": "started"}, request)
@app.route('/api/nettest', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_nettest_list(self, request):
return self.render_json(self.director.netTests, request)
@app.route('/api/input', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_input_list(self, request):
input_store_list = self.director.input_store.list()
for key, value in input_store_list.items():
value.pop('filepath')
return self.render_json(input_store_list, request)
@app.route('/api/input/<string:input_id>/content', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_input_content(self, request, input_id):
content = self.director.input_store.getContent(input_id)
request.setHeader('Content-Type', 'text/plain')
request.setHeader('Content-Length', len(content))
return content
@app.route('/api/input/<string:input_id>', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_input_details(self, request, input_id):
input_desc = self.director.input_store.get(input_id)
input_desc.pop('filepath')
return self.render_json(input_desc, request)
@app.route('/api/measurement', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_measurement_list(self, request):
measurements = list_measurements(order='desc')
for measurement in measurements:
if measurement['running'] == False:
continue
try:
net_test = self.director.activeMeasurements[measurement['id']]
measurement['progress'] = net_test.completionPercentage * 100
except KeyError:
log.err("Did not find measurement with ID %s" %
measurement['id'])
return self.render_json({"measurements": measurements}, request)
@app.route('/api/measurement/<string:measurement_id>', methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
@defer.inlineCallbacks
def api_measurement_summary(self, request, measurement_id):
try:
measurement = get_measurement(measurement_id)
except InsecurePath:
raise WebUIError(500, "invalid measurement id")
except MeasurementNotFound:
raise WebUIError(404, "measurement not found")
except MeasurementInProgress:
raise WebUIError(400, "measurement in progress")
if measurement['completed'] is False:
raise WebUIError(400, "measurement in progress")
summary = yield get_summary(measurement_id)
defer.returnValue(self.render_json(summary, request))
@app.route('/api/measurement/<string:measurement_id>', methods=["DELETE"])
@xsrf_protect(check=True)
@requires_true(attrs=['_is_initialized'])
def api_measurement_delete(self, request, measurement_id):
try:
measurement = get_measurement(measurement_id)
except InsecurePath:
raise WebUIError(500, "invalid measurement id")
except MeasurementNotFound:
raise WebUIError(404, "measurement not found")
if measurement['running'] is True:
raise WebUIError(400, "Measurement running")
try:
measurement_dir = self.measurement_path.child(measurement_id)
measurement_dir.remove()
except:
raise WebUIError(400, "Failed to delete report")
return self.render_json({"result": "ok"}, request)
@app.route('/api/measurement/<string:measurement_id>/keep',
methods=["POST"])
@xsrf_protect(check=True)
@requires_true(attrs=['_is_initialized'])
def api_measurement_keep(self, request, measurement_id):
try:
measurement_dir = self.measurement_path.child(measurement_id)
except InsecurePath:
raise WebUIError(500, "invalid measurement id")
summary = measurement_dir.child("keep")
with summary.open("w+") as f:
pass
return self.render_json({"status": "ok"}, request)
@app.route('/api/measurement/<string:measurement_id>/<int:idx>',
methods=["GET"])
@xsrf_protect(check=False)
@requires_true(attrs=['_is_initialized'])
def api_measurement_view(self, request, measurement_id, idx):
try:
measurement_dir = self.measurement_path.child(measurement_id)
except InsecurePath:
raise WebUIError(500, "Invalid measurement id")
measurements = measurement_dir.child("measurements.njson")
# This gets the line idx of the measurement file.
# XXX maybe implement some caching here
with measurements.open("r") as f:
r = None
for f_idx, line in enumerate(f):
if f_idx == idx:
r = json.loads(line)
break
if r is None:
raise WebUIError(404,
"Could not find measurement with this idx")
return self.render_json(r, request)
@app.route('/api/logs', methods=["GET"])
@xsrf_protect(check=True)
@requires_true(attrs=['_is_initialized'])
def api_get_logs(self, request):
with open(log.oonilogger.log_filepath) as input_file:
log_data = input_file.read()
logs = {'latest': log_data, 'older': []}
if request.args.get('all', False) is not False:
for log_filepath in glob(log.oonilogger.log_filepath + ".*"):
with open(log_filepath) as input_file:
log_data = input_file.read()
logs['older'].append(log_data)
logs['older'].reverse()
return self.render_json(logs, request)
@app.route('/client/', branch=True)
@xsrf_protect(check=False)
def static(self, request):
return static.File(config.web_ui_directory)
