def gotHasKey(result):
if result:
if not self.hasHostKey(ip, key):
ui.warn("Warning: Permanently added the %s host key for "
"IP address '%s' to the list of known hosts." %
(key.type(), nativeString(ip)))
self.addHostKey(ip, key)
self.save()
return result
else:
def promptResponse(response):
if response:
self.addHostKey(hostname, key)
self.addHostKey(ip, key)
self.save()
return response
else:
raise UserRejectedKey()
keytype = key.type()
if keytype is "EC":
keytype = "ECDSA"
prompt = (
"The authenticity of host '%s (%s)' "
"can't be established.\n"
"%s key fingerprint is SHA256:%s.\n"
"Are you sure you want to continue connecting (yes/no)? " %
(nativeString(hostname), nativeString(ip), keytype,
key.fingerprint(format=FingerprintFormats.SHA256_BASE64)))
proceed = ui.prompt(prompt.encode(sys.getdefaultencoding()))
return proceed.addCallback(promptResponse)
def uri(self):
# type: () -> URL
request = self._request
# This code borrows from t.w.server.Request._prePathURL.
if request.isSecure():
scheme = u"https"
else:
scheme = u"http"
netloc = nativeString(request.getRequestHostname())
port = request.getHost().port
if request.isSecure():
default = 443
else:
default = 80
if port != default:
netloc += u":{}".format(port)
path = nativeString(request.uri)
if path and path[0] == u"/":
path = path[1:]
return URL.fromText(u"{}://{}/{}".format(scheme, netloc, path))
def lineReceived(self, rawSentence):
"""
Parses the data from the sentence and validates the checksum.
@param rawSentence: The NMEA positioning sentence.
@type rawSentence: C{bytes}
"""
sentence = rawSentence.strip()
_validateChecksum(sentence)
splitSentence = _split(sentence)
sentenceType = nativeString(splitSentence[0])
contents = [nativeString(x) for x in splitSentence[1:]]
try:
keys = self._SENTENCE_CONTENTS[sentenceType]
except KeyError:
raise ValueError("unknown sentence type %s" % sentenceType)
sentenceData = {"type": sentenceType}
for key, value in izip(keys, contents):
if key is not None and value != "":
sentenceData[key] = value
sentence = NMEASentence(sentenceData)
if self._sentenceCallback is not None:
self._sentenceCallback(sentence)
self._receiver.sentenceReceived(sentence)
def callRemote(self, method, *args):
"""
Call remote XML-RPC C{method} with given arguments.
@return: a L{defer.Deferred} that will fire with the method response,
or a failure if the method failed. Generally, the failure type will
be L{Fault}, but you can also have an C{IndexError} on some buggy
servers giving empty responses.
If the deferred is cancelled before the request completes, the
connection is closed and the deferred will fire with a
L{defer.CancelledError}.
"""
def cancel(d):
factory.deferred = None
connector.disconnect()
factory = self.queryFactory(
self.path, self.host, method, self.user,
self.password, self.allowNone, args, cancel, self.useDateTime)
if self.secure:
from twisted.internet import ssl
connector = self._reactor.connectSSL(
nativeString(self.host), self.port or 443,
factory, ssl.ClientContextFactory(),
timeout=self.connectTimeout)
else:
connector = self._reactor.connectTCP(
nativeString(self.host), self.port or 80, factory,
timeout=self.connectTimeout)
return factory.deferred
def filenameToModuleName(fn):
"""
Convert a name in the filesystem to the name of the Python module it is.
This is aggressive about getting a module name back from a file; it will
always return a string. Aggressive means 'sometimes wrong'; it won't look
at the Python path or try to do any error checking: don't use this method
unless you already know that the filename you're talking about is a Python
module.
@param fn: A filesystem path to a module or package; C{bytes} on Python 2,
C{bytes} or C{unicode} on Python 3.
@return: A hopefully importable module name.
@rtype: C{str}
"""
if isinstance(fn, bytes):
initPy = b"__init__.py"
else:
initPy = "__init__.py"
fullName = os.path.abspath(fn)
base = os.path.basename(fn)
if not base:
# this happens when fn ends with a path separator, just skit it
base = os.path.basename(fn[:-1])
modName = nativeString(os.path.splitext(base)[0])
while 1:
fullName = os.path.dirname(fullName)
if os.path.exists(os.path.join(fullName, initPy)):
modName = "%s.%s" % (
nativeString(os.path.basename(fullName)),
nativeString(modName))
else:
break
return modName
def lineReceived(self, line):
"""
Receive line commands from the server.
"""
self.resetTimeout()
token = line.split(b" ", 1)[0]
# First manage standard commands without space
cmd = getattr(self, "cmd_" + nativeString(token), None)
if cmd is not None:
args = line.split(b" ", 1)[1:]
if args:
cmd(args[0])
else:
cmd()
else:
# Then manage commands with space in it
line = line.replace(b" ", b"_")
cmd = getattr(self, "cmd_" + nativeString(line), None)
if cmd is not None:
cmd()
else:
# Increment/Decrement response
cmd = self._current.popleft()
val = int(line)
cmd.success(val)
if not self._current:
# No pending request, remove timeout
self.setTimeout(None)
def test_authInfoInURL(self):
url = "http://%s:%[email protected]:%d/" % (
nativeString(self.user), nativeString(self.password), self.port)
p = xmlrpc.Proxy(networkString(url))
d = p.callRemote("authinfo")
d.addCallback(self.assertEqual, [self.user, self.password])
return d
def class_IN(self, ttl, type, domain, rdata):
"""
Simulate a class IN and recurse into the actual class.
@param ttl: time to live for the record
@type ttl: L{int}
@param type: record type
@type type: str
@param domain: the domain
@type domain: bytes
@param rdata:
@type rdate: bytes
"""
record = getattr(dns, 'Record_%s' % (nativeString(type),), None)
if record:
r = record(*rdata)
r.ttl = ttl
self.records.setdefault(domain.lower(), []).append(r)
if type == 'SOA':
self.soa = (domain, r)
else:
raise NotImplementedError(
"Record type %r not supported" % (nativeString(type),)
)
def assertNativeString(self, original, expected):
"""
Raise an exception indicating a failed test if the output of
C{nativeString(original)} is unequal to the expected string, or is not
a native string.
"""
self.assertEqual(nativeString(original), expected)
self.assertIsInstance(nativeString(original), str)
def _unjelly_function(self, rest):
fname = nativeString(rest[0])
modSplit = fname.split(nativeString('.'))
modName = nativeString('.').join(modSplit[:-1])
if not self.taster.isModuleAllowed(modName):
raise InsecureJelly("Module not allowed: %s" % modName)
# XXX do I need an isFunctionAllowed?
function = namedAny(fname)
return function
def posixGetLinkLocalIPv6Addresses():
"""
Return a list of strings in colon-hex format representing all the link local
IPv6 addresses available on the system, as reported by I{getifaddrs(3)}.
"""
retList = []
for (interface, family, address) in _interfaces():
interface = nativeString(interface)
address = nativeString(address)
if family == socket.AF_INET6 and address.startswith('fe80:'):
retList.append('%s%%%s' % (address, interface))
return retList
def _ip(src, dst, payload):
"""
Construct an IP datagram with the given source, destination, and
application payload.
@param src: The source IPv4 address as a dotted-quad string.
@type src: L{bytes}
@param dst: The destination IPv4 address as a dotted-quad string.
@type dst: L{bytes}
@param payload: The content of the IP datagram (such as a UDP datagram).
@type payload: L{bytes}
@return: An IP datagram header and payload.
@rtype: L{bytes}
"""
ipHeader = (
# Version and header length, 4 bits each
b'\x45'
# Differentiated services field
b'\x00'
# Total length
+ _H(20 + len(payload))
+ b'\x00\x01\x00\x00\x40\x11'
# Checksum
+ _H(0)
# Source address
+ socket.inet_pton(socket.AF_INET, nativeString(src))
# Destination address
+ socket.inet_pton(socket.AF_INET, nativeString(dst)))
# Total all of the 16-bit integers in the header
checksumStep1 = sum(struct.unpack('!10H', ipHeader))
# Pull off the carry
carry = checksumStep1 >> 16
# And add it to what was left over
checksumStep2 = (checksumStep1 & 0xFFFF) + carry
# Compute the one's complement sum
checksumStep3 = checksumStep2 ^ 0xFFFF
# Reconstruct the IP header including the correct checksum so the platform
# IP stack, if there is one involved in this test, doesn't drop it on the
# floor as garbage.
ipHeader = (
ipHeader[:10] +
struct.pack('!H', checksumStep3) +
ipHeader[12:])
return ipHeader + payload
def _unjelly_class(self, rest):
cname = nativeString(rest[0])
clist = cname.split(nativeString('.'))
modName = nativeString('.').join(clist[:-1])
if not self.taster.isModuleAllowed(modName):
raise InsecureJelly("module %s not allowed" % modName)
klaus = namedObject(cname)
objType = type(klaus)
if objType not in (_OldStyleClass, type):
raise InsecureJelly(
"class %r unjellied to something that isn't a class: %r" % (
cname, klaus))
if not self.taster.isClassAllowed(klaus):
raise InsecureJelly("class not allowed: %s" % qual(klaus))
return klaus
def gotHasKey(result):
if result:
if not self.knownHosts.hasHostKey(ip, key):
log.msg("Added new %s host key for IP address '%s'." %
(key.type(), nativeString(ip)))
self.knownHosts.addHostKey(ip, key)
self.knownHosts.save()
return result
else:
log.msg("Added %s host key for IP address '%s'." %
(key.type(), nativeString(ip)))
self.knownHosts.addHostKey(hostname, key)
self.knownHosts.addHostKey(ip, key)
self.knownHosts.save()
return True
def getPassword(self, prompt = None):
if prompt:
prompt = nativeString(prompt)
else:
prompt = ("%s@%s's password: " %
(nativeString(self.user), self.transport.transport.getPeer().host))
try:
# We don't know the encoding the other side is using,
# signaling that is not part of the SSH protocol. But
# using our defaultencoding is better than just going for
# ASCII.
p = self._getPassword(prompt).encode(sys.getdefaultencoding())
return defer.succeed(p)
except ConchError:
return defer.fail()
def test_formatRootShortUnicodeString(self):
"""
The C{_formatRoot} method formats a short unicode string using the
built-in repr.
"""
e = self.makeFlattenerError()
self.assertEqual(e._formatRoot(nativeString('abcd')), repr('abcd'))
def render(self, request):
"""
Render a given resource. See L{IResource}'s render method.
I delegate to methods of self with the form 'render_METHOD'
where METHOD is the HTTP that was used to make the
request. Examples: render_GET, render_HEAD, render_POST, and
so on. Generally you should implement those methods instead of
overriding this one.
render_METHOD methods are expected to return a byte string which will be
the rendered page, unless the return value is C{server.NOT_DONE_YET}, in
which case it is this class's responsibility to write the results using
C{request.write(data)} and then call C{request.finish()}.
Old code that overrides render() directly is likewise expected
to return a byte string or NOT_DONE_YET.
@see: L{IResource.render}
"""
m = getattr(self, 'render_' + nativeString(request.method), None)
if not m:
try:
allowedMethods = self.allowedMethods
except AttributeError:
allowedMethods = _computeAllowedMethods(self)
raise UnsupportedMethod(allowedMethods)
return m(request)
def _inspect(self):
return '\n'.join(['Certificate For Subject:',
self.getSubject().inspect(),
'\nIssuer:',
self.getIssuer().inspect(),
'\nSerial Number: %d' % self.serialNumber(),
'Digest: %s' % nativeString(self.digest())])
def __str__(self):
"""
Return a string representation of this L{SerialNumber} instance.
@rtype: L{nativeString}
"""
return nativeString('%d' % (self._number,))
def __repr__(self):
"""
Return a pretty representation of this object.
"""
lines = [
'<%s %s (%s bits)' % (
nativeString(self.type()),
self.isPublic() and 'Public Key' or 'Private Key',
self._keyObject.key_size)]
for k, v in sorted(self.data().items()):
if _PY3 and isinstance(k, bytes):
k = k.decode('ascii')
lines.append('attr %s:' % (k,))
by = common.MP(v)[4:]
while by:
m = by[:15]
by = by[15:]
o = ''
for c in iterbytes(m):
o = o + '%02x:' % (ord(c),)
if len(m) < 15:
o = o[:-1]
lines.append('\t' + o)
lines[-1] = lines[-1] + '>'
return '\n'.join(lines)
def testFactoryInfo(self):
url = self.getURL("file")
uri = client.URI.fromBytes(url)
factory = client.HTTPClientFactory(url)
reactor.connectSSL(nativeString(uri.host), uri.port, factory, ssl.ClientContextFactory())
# The base class defines _cbFactoryInfo correctly for this
return factory.deferred.addCallback(self._cbFactoryInfo, factory)
def gotGlobalRequest(self, requestType, data):
# XXX should this use method dispatch?
requestType = nativeString(requestType.replace(b'-', b'_'))
f = getattr(self, "global_%s" % requestType, None)
if not f:
return 0
return f(data)
def tryAuth(self, kind, user, data):
"""
Try to authenticate the user with the given method. Dispatches to a
auth_* method.
@param kind: the authentication method to try.
@type kind: L{bytes}
@param user: the username the client is authenticating with.
@type user: L{bytes}
@param data: authentication specific data sent by the client.
@type data: L{bytes}
@return: A Deferred called back if the method succeeded, or erred back
if it failed.
@rtype: C{defer.Deferred}
"""
log.msg('%r trying auth %r' % (user, kind))
if kind not in self.supportedAuthentications:
return defer.fail(
error.ConchError('unsupported authentication, failing'))
kind = nativeString(kind.replace(b'-', b'_'))
f = getattr(self, 'auth_%s' % (kind,), None)
if f:
ret = f(data)
if not ret:
return defer.fail(
error.ConchError(
'%s return None instead of a Deferred'
% (kind, )))
else:
return ret
return defer.fail(error.ConchError('bad auth type: %s' % (kind,)))
def _expandCipherString(cipherString, method, options):
"""
Expand C{cipherString} according to C{method} and C{options} to a list
of explicit ciphers that are supported by the current platform.
@param cipherString: An OpenSSL cipher string to expand.
@type cipherString: L{unicode}
@param method: An OpenSSL method like C{SSL.TLSv1_METHOD} used for
determining the effective ciphers.
@param options: OpenSSL options like C{SSL.OP_NO_SSLv3} ORed together.
@type options: L{int}
@return: The effective list of explicit ciphers that results from the
arguments on the current platform.
@rtype: L{list} of L{ICipher}
"""
ctx = SSL.Context(method)
ctx.set_options(options)
try:
ctx.set_cipher_list(nativeString(cipherString))
except SSL.Error as e:
if e.args[0][0][2] == 'no cipher match':
return []
else:
raise
conn = SSL.Connection(ctx, None)
ciphers = conn.get_cipher_list()
if isinstance(ciphers[0], unicode):
return [OpenSSLCipher(cipher) for cipher in ciphers]
else:
return [OpenSSLCipher(cipher.decode('ascii')) for cipher in ciphers]
def test_repr(self):
"""
The repr of L{UNIXAddress} returns with the filename that the
L{UNIXAddress} is for.
"""
self.assertEqual(repr(self.buildAddress()), "UNIXAddress('%s')" % (
nativeString(self._socketAddress)))
def unjelly(self, obj):
if type(obj) is not list:
return obj
jelTypeBytes = obj[0]
if not self.taster.isTypeAllowed(jelTypeBytes):
raise InsecureJelly(jelTypeBytes)
regClass = unjellyableRegistry.get(jelTypeBytes)
if regClass is not None:
method = getattr(_createBlank(regClass), "unjellyFor", regClass)
return self._maybePostUnjelly(method(self, obj))
regFactory = unjellyableFactoryRegistry.get(jelTypeBytes)
if regFactory is not None:
return self._maybePostUnjelly(regFactory(self.unjelly(obj[1])))
jelTypeText = nativeString(jelTypeBytes)
thunk = getattr(self, '_unjelly_%s' % jelTypeText, None)
if thunk is not None:
return thunk(obj[1:])
else:
nameSplit = jelTypeText.split('.')
modName = '.'.join(nameSplit[:-1])
if not self.taster.isModuleAllowed(modName):
raise InsecureJelly(
"Module %s not allowed (in type %s)." % (modName, jelTypeText))
clz = namedObject(jelTypeText)
if not self.taster.isClassAllowed(clz):
raise InsecureJelly("Class %s not allowed." % jelTypeText)
return self._genericUnjelly(clz, obj[1])
def __init__(self, snName):
"""
@param snName: The name of the curve as used by C{OBJ_sn2nid}.
@param snName: L{unicode}
@raises NotImplementedError: If ECC support is not available.
@raises ValueError: If C{snName} is not a supported curve.
"""
self.snName = nativeString(snName)
# As soon as pyOpenSSL supports ECDHE directly, attempt to use its
# APIs first. See #7033.
# If pyOpenSSL is based on cryptography.io (0.14+), we use its
# bindings directly to set the ECDHE curve.
try:
binding = self._getBinding()
self._lib = binding.lib
self._ffi = binding.ffi
self._nid = self._lib.OBJ_sn2nid(self.snName.encode('ascii'))
if self._nid == self._lib.NID_undef:
raise ValueError("Unknown ECC curve.")
except AttributeError:
raise NotImplementedError(
"This version of pyOpenSSL does not support ECC."
)
def _parseAttributes(self, data):
flags ,= struct.unpack('!L', data[:4])
attrs = {}
data = data[4:]
if flags & FILEXFER_ATTR_SIZE == FILEXFER_ATTR_SIZE:
size ,= struct.unpack('!Q', data[:8])
attrs['size'] = size
data = data[8:]
if flags & FILEXFER_ATTR_OWNERGROUP == FILEXFER_ATTR_OWNERGROUP:
uid, gid = struct.unpack('!2L', data[:8])
attrs['uid'] = uid
attrs['gid'] = gid
data = data[8:]
if flags & FILEXFER_ATTR_PERMISSIONS == FILEXFER_ATTR_PERMISSIONS:
perms ,= struct.unpack('!L', data[:4])
attrs['permissions'] = perms
data = data[4:]
if flags & FILEXFER_ATTR_ACMODTIME == FILEXFER_ATTR_ACMODTIME:
atime, mtime = struct.unpack('!2L', data[:8])
attrs['atime'] = atime
attrs['mtime'] = mtime
data = data[8:]
if flags & FILEXFER_ATTR_EXTENDED == FILEXFER_ATTR_EXTENDED:
extended_count ,= struct.unpack('!L', data[:4])
data = data[4:]
for i in xrange(extended_count):
extended_type, data = getNS(data)
extended_data, data = getNS(data)
attrs['ext_%s' % nativeString(extended_type)] = extended_data
return attrs, data
def _is_valid_keyblob(key_blob_or_type):
try:
key_blob_or_type = nativeString(key_blob_or_type)
except (UnicodeError, TypeError):
return False
else:
return re.match(r'[^ :]+:[^ :]+$', key_blob_or_type)
def searchFileForAll(hostsFile, name):
"""
Search the given file, which is in hosts(5) standard format, for an address
entry with a given name.
@param hostsFile: The name of the hosts(5)-format file to search.
@type hostsFile: L{FilePath}
@param name: The name to search for.
@type name: C{str}
@return: C{None} if the name is not found in the file, otherwise a
C{str} giving the address in the file associated with the name.
"""
results = []
try:
lines = hostsFile.getContent().splitlines()
except:
return results
name = name.lower()
for line in lines:
idx = line.find(b'#')
if idx != -1:
line = line[:idx]
if not line:
continue
parts = line.split()
if name.lower() in [s.lower() for s in parts[1:]]:
results.append(nativeString(parts[0]))
return results
def render(self, resrc):
"""
Ask a resource to render itself.
@param resrc: a L{twisted.web.resource.IResource}.
"""
try:
body = resrc.render(self)
except UnsupportedMethod as e:
allowedMethods = e.allowedMethods
if (self.method == b"HEAD") and (b"GET" in allowedMethods):
# We must support HEAD (RFC 2616, 5.1.1). If the
# resource doesn't, fake it by giving the resource
# a 'GET' request and then return only the headers,
# not the body.
self._log.info("Using GET to fake a HEAD request for {resrc}",
resrc=resrc)
self.method = b"GET"
self._inFakeHead = True
body = resrc.render(self)
if body is NOT_DONE_YET:
self._log.info(
"Tried to fake a HEAD request for {resrc}, but "
"it got away from me.",
resrc=resrc)
# Oh well, I guess we won't include the content length.
else:
self.setHeader(b'content-length', intToBytes(len(body)))
self._inFakeHead = False
self.method = b"HEAD"
self.write(b'')
self.finish()
return
if self.method in (supportedMethods):
# We MUST include an Allow header
# (RFC 2616, 10.4.6 and 14.7)
self.setHeader(b'Allow', b', '.join(allowedMethods))
s = ('''Your browser approached me (at %(URI)s) with'''
''' the method "%(method)s". I only allow'''
''' the method%(plural)s %(allowed)s here.''' % {
'URI':
escape(nativeString(self.uri)),
'method':
nativeString(self.method),
'plural': ((len(allowedMethods) > 1) and 's') or '',
'allowed':
', '.join([nativeString(x) for x in allowedMethods])
})
epage = resource.ErrorPage(http.NOT_ALLOWED,
"Method Not Allowed", s)
body = epage.render(self)
else:
epage = resource.ErrorPage(
http.NOT_IMPLEMENTED, "Huh?",
"I don't know how to treat a %s request." %
(escape(self.method.decode("charmap")), ))
body = epage.render(self)
# end except UnsupportedMethod
if body == NOT_DONE_YET:
return
if not isinstance(body, bytes):
body = resource.ErrorPage(
http.INTERNAL_SERVER_ERROR, "Request did not return bytes",
"Request: " + util._PRE(reflect.safe_repr(self)) + "<br />" +
"Resource: " + util._PRE(reflect.safe_repr(resrc)) + "<br />" +
"Value: " + util._PRE(reflect.safe_repr(body))).render(self)
if self.method == b"HEAD":
if len(body) > 0:
# This is a Bad Thing (RFC 2616, 9.4)
self._log.info(
"Warning: HEAD request {slf} for resource {resrc} is"
" returning a message body. I think I'll eat it.",
slf=self,
resrc=resrc)
self.setHeader(b'content-length', intToBytes(len(body)))
self.write(b'')
else:
self.setHeader(b'content-length', intToBytes(len(body)))
self.write(body)
self.finish()
def logPrefix(self):
id = (self.id is not None and str(self.id)) or "unknown"
name = self.name
if name:
name = nativeString(name)
return "SSHChannel %s (%s) on %s" % (name, id, self.conn.logPrefix())
def getExpectedStartListeningLogMessage(self, port, factory):
"""
Get the message expected to be logged when a UNIX port starts listening.
"""
return "%s starting on %r" % (factory, nativeString(
port.getHost().name))
def _flattenElement(request, root, write, slotData, renderFactory,
dataEscaper):
"""
Make C{root} slightly more flat by yielding all its immediate contents as
strings, deferreds or generators that are recursive calls to itself.
@param request: A request object which will be passed to
L{IRenderable.render}.
@param root: An object to be made flatter. This may be of type C{unicode},
L{str}, L{slot}, L{Tag <twisted.web.template.Tag>}, L{tuple}, L{list},
L{types.GeneratorType}, L{Deferred}, or an object that implements
L{IRenderable}.
@param write: A callable which will be invoked with each L{bytes} produced
by flattening C{root}.
@param slotData: A L{list} of L{dict} mapping L{str} slot names to data
with which those slots will be replaced.
@param renderFactory: If not L{None}, an object that provides
L{IRenderable}.
@param dataEscaper: A 1-argument callable which takes L{bytes} or
L{unicode} and returns L{bytes}, quoted as appropriate for the
rendering context. This is really only one of two values:
L{attributeEscapingDoneOutside} or L{escapeForContent}, depending on
whether the rendering context is within an attribute or not. See the
explanation in L{writeWithAttributeEscaping}.
@return: An iterator that eventually yields L{bytes} that should be written
to the output. However it may also yield other iterators or
L{Deferred}s; if it yields another iterator, the caller will iterate
it; if it yields a L{Deferred}, the result of that L{Deferred} will
either be L{bytes}, in which case it's written, or another generator,
in which case it is iterated. See L{_flattenTree} for the trampoline
that consumes said values.
@rtype: An iterator which yields L{bytes}, L{Deferred}, and more iterators
of the same type.
"""
def keepGoing(newRoot,
dataEscaper=dataEscaper,
renderFactory=renderFactory,
write=write):
return _flattenElement(request, newRoot, write, slotData,
renderFactory, dataEscaper)
if isinstance(root, (bytes, unicode)):
write(dataEscaper(root))
elif isinstance(root, slot):
slotValue = _getSlotValue(root.name, slotData, root.default)
yield keepGoing(slotValue)
elif isinstance(root, CDATA):
write(b'<![CDATA[')
write(escapedCDATA(root.data))
write(b']]>')
elif isinstance(root, Comment):
write(b'<!--')
write(escapedComment(root.data))
write(b'-->')
elif isinstance(root, Tag):
slotData.append(root.slotData)
if root.render is not None:
rendererName = root.render
rootClone = root.clone(False)
rootClone.render = None
renderMethod = renderFactory.lookupRenderMethod(rendererName)
result = renderMethod(request, rootClone)
yield keepGoing(result)
slotData.pop()
return
if not root.tagName:
yield keepGoing(root.children)
return
write(b'<')
if isinstance(root.tagName, unicode):
tagName = root.tagName.encode('ascii')
else:
tagName = root.tagName
write(tagName)
for k, v in iteritems(root.attributes):
if isinstance(k, unicode):
k = k.encode('ascii')
write(b' ' + k + b'="')
# Serialize the contents of the attribute, wrapping the results of
# that serialization so that _everything_ is quoted.
yield keepGoing(v,
attributeEscapingDoneOutside,
write=writeWithAttributeEscaping(write))
write(b'"')
if root.children or nativeString(tagName) not in voidElements:
write(b'>')
# Regardless of whether we're in an attribute or not, switch back
# to the escapeForContent dataEscaper. The contents of a tag must
# be quoted no matter what; in the top-level document, just so
# they're valid, and if they're within an attribute, they have to
# be quoted so that after applying the *un*-quoting required to re-
# parse the tag within the attribute, all the quoting is still
# correct.
yield keepGoing(root.children, escapeForContent)
write(b'</' + tagName + b'>')
else:
write(b' />')
elif isinstance(root, (tuple, list, GeneratorType)):
for element in root:
yield keepGoing(element)
elif isinstance(root, CharRef):
escaped = '&#%d;' % (root.ordinal, )
write(escaped.encode('ascii'))
elif isinstance(root, Deferred):
yield root.addCallback(lambda result: (result, keepGoing(result)))
elif iscoroutine(root):
d = ensureDeferred(root)
yield d.addCallback(lambda result: (result, keepGoing(result)))
elif IRenderable.providedBy(root):
result = root.render(request)
yield keepGoing(result, renderFactory=root)
else:
raise UnsupportedType(root)
def testFactoryInfo(self):
url = self.getURL('file')
uri = client._URI.fromBytes(url)
factory = client.HTTPClientFactory(url)
reactor.connectTCP(nativeString(uri.host), uri.port, factory)
return factory.deferred.addCallback(self._cbFactoryInfo, factory)
def create_global_configuration(basedir, api_endpoint_str,
tahoe_node_directory, api_client_endpoint_str):
"""
Create a new global configuration in `basedir` (which must not yet exist).
:param FilePath basedir: a non-existant directory
:param unicode api_endpoint_str: the Twisted server endpoint string
where we will listen for API requests.
:param FilePath tahoe_node_directory: the directory our Tahoe LAFS
client uses.
:param unicode api_client_endpoint_str: the Twisted client endpoint
string where our API can be contacted.
:returns: a GlobalConfigDatabase instance
"""
# our APIs insist on endpoint-strings being unicode, but Twisted
# only accepts "str" .. so we have to convert on py2. When we
# support python3 this check only needs to happen on py2
if not isinstance(api_endpoint_str, unicode):
raise ValueError("'api_endpoint_str' must be unicode")
if not isinstance(api_client_endpoint_str, unicode):
raise ValueError("'api_client_endpoint_str' must be unicode")
api_endpoint_str = nativeString(api_endpoint_str)
api_client_endpoint_str = nativeString(api_client_endpoint_str)
# check that the endpoints are valid (will raise exception if not)
_validate_listen_endpoint_str(api_endpoint_str)
_validate_connect_endpoint_str(api_client_endpoint_str)
# note that we put *bytes* in .child() calls after this so we
# don't convert again..
basedir = basedir.asBytesMode("utf8")
try:
basedir.makedirs()
except OSError as e:
raise ValueError("'{}' already exists: {}".format(basedir.path, e))
# explain what is in this directory
with basedir.child(b"README").open("wb") as f:
f.write(u"This is a Magic Folder daemon configuration\n"
u"\n"
u"To find out more you can run a command like:\n"
u"\n"
u" magic-folder --config {} --help\n"
u"\n".format(basedir.asTextMode("utf8").path).encode("utf8"))
# set up the configuration database
db_fname = basedir.child(b"global.sqlite")
connection = _upgraded(
_global_config_schema,
sqlite3.connect(db_fname.path),
)
with connection:
cursor = connection.cursor()
cursor.execute(
"INSERT INTO config (api_endpoint, tahoe_node_directory, api_client_endpoint) VALUES (?, ?, ?)",
(api_endpoint_str, tahoe_node_directory.path,
api_client_endpoint_str))
config = GlobalConfigDatabase(basedir=basedir,
database=connection,
token_provider=FilesystemTokenProvider(
basedir.child(b"api_token"), ))
# make sure we have an API token
config.rotate_api_token()
return config
def _absPath(self, path):
home = self.avatar.getHomeDir()
return os.path.join(nativeString(home.path), nativeString(path))
def _absPath(self, path):
home = self.avatar.home
return os.path.abspath(
os.path.join(nativeString(home), nativeString(path)))
# Copyright (c) Twisted Matrix Laboratories.
# See LICENSE for details.
"""
Helper classes for twisted.test.test_ssl.
They are in a separate module so they will not prevent test_ssl importing if
pyOpenSSL is unavailable.
"""
from __future__ import division, absolute_import
from OpenSSL import SSL
from twisted.internet import ssl
from twisted.python.compat import nativeString
from twisted.python.filepath import FilePath
certPath = nativeString(
FilePath(__file__.encode("utf-8")).sibling(b"server.pem").path)
class ClientTLSContext(ssl.ClientContextFactory):
isClient = 1
def getContext(self):
return SSL.Context(SSL.TLSv1_METHOD)
class ServerTLSContext:
isClient = 0
def __init__(self, filename=certPath, method=SSL.TLSv1_METHOD):
self.filename = filename
self._method = method
def getExpectedConnectionLostLogMsg(self, port):
"""
Get the expected connection lost message for a UNIX port
"""
return "(UNIX Port %s Closed)" % (nativeString(port.getHost().name), )
def __str__(self) -> str:
return nativeString(self.status + b" " + self.message)
def getURL(self, path):
host = "http://127.0.0.1:%d/" % self.portno
return networkString(urljoin(host, nativeString(path)))
def getpass(p):
self.assertEqual(p, nativeString(prompt))
return 'bad password'
from twisted.trial import unittest
from twisted.web import server, client, error, resource
from twisted.internet import reactor, defer, interfaces
from twisted.python.filepath import FilePath
from twisted.python.log import msg
from twisted.protocols.policies import WrappingFactory
from twisted.test.proto_helpers import StringTransport
try:
from twisted.internet import ssl
except:
ssl = None
from twisted import test
serverPEM = FilePath(test.__file__.encode("utf-8")).sibling(b'server.pem')
serverPEMPath = nativeString(serverPEM.path)
# Remove this in #6177, when static is ported to Python 3:
if _PY3:
from twisted.web.test.test_web import Data
else:
from twisted.web.static import Data
# Remove this in #6178, when util is ported to Python 3:
if _PY3:
class Redirect(resource.Resource):
isLeaf = 1
def __init__(self, url):
resource.Resource.__init__(self)
def setResponse(self, response):
setattr(self, nativeString(self.responses.pop()), response)
def __init__(self, templates, directory):
Resource.__init__(self)
self._page = templates.get_template('cb_web_404.html')
self._directory = nativeString(directory)
def _getPassword(prompt):
self.assertEqual(
prompt,
"Enter passphrase for key '%s': " % (self.rsaFile.path, ))
return nativeString(passphrase)
def __str__(self):
return nativeString(self.__bytes__())
def _getPassword(prompt):
self.assertEqual(
prompt, f"Enter passphrase for key '{self.rsaFile.path}': ")
return nativeString(passphrase)
def render(self, request):
header = request.getHeader(b'cookie')
if header is None:
return b'None'
return networkString(repr(nativeString(header)))
def parseRecordLine(self, origin, ttl, line):
"""
Parse a C{line} from a zone file respecting C{origin} and C{ttl}.
Add resulting records to authority.
@param origin: starting point for the zone
@type origin: L{bytes}
@param ttl: time to live for the record
@type ttl: L{int}
@param line: zone file line to parse; split by word
@type line: L{list} of L{bytes}
"""
if _PY3:
queryClasses = set(
qc.encode("ascii") for qc in dns.QUERY_CLASSES.values())
queryTypes = set(
qt.encode("ascii") for qt in dns.QUERY_TYPES.values())
else:
queryClasses = set(dns.QUERY_CLASSES.values())
queryTypes = set(dns.QUERY_TYPES.values())
markers = queryClasses | queryTypes
cls = b'IN'
owner = origin
if line[0] == b'@':
line = line[1:]
owner = origin
elif not line[0].isdigit() and line[0] not in markers:
owner = line[0]
line = line[1:]
if line[0].isdigit() or line[0] in markers:
domain = owner
owner = origin
else:
domain = line[0]
line = line[1:]
if line[0] in queryClasses:
cls = line[0]
line = line[1:]
if line[0].isdigit():
ttl = int(line[0])
line = line[1:]
elif line[0].isdigit():
ttl = int(line[0])
line = line[1:]
if line[0] in queryClasses:
cls = line[0]
line = line[1:]
type = line[0]
rdata = line[1:]
self.addRecord(owner, ttl, nativeString(type), domain,
nativeString(cls), rdata)
def _doMultipleRangeRequest(self, request, byteRanges):
"""
Set up the response for Range headers that specify a single range.
This method checks if the request is satisfiable and sets the response
code and Content-Type and Content-Length headers appropriately. The
return value, which is a little complicated, indicates which parts of
the resource to return and the boundaries that should separate the
parts.
In detail, the return value is a tuple rangeInfo C{rangeInfo} is a
list of 3-tuples C{(partSeparator, partOffset, partSize)}. The
response to this request should be, for each element of C{rangeInfo},
C{partSeparator} followed by C{partSize} bytes of the resource
starting at C{partOffset}. Each C{partSeparator} includes the
MIME-style boundary and the part-specific Content-type and
Content-range headers. It is convenient to return the separator as a
concrete string from this method, because this method needs to compute
the number of bytes that will make up the response to be able to set
the Content-Length header of the response accurately.
@param request: The Request object.
@param byteRanges: A list of C{(start, end)} values as specified by
the header. For each range, at most one of C{start} and C{end}
may be C{None}.
@return: See above.
"""
matchingRangeFound = False
rangeInfo = []
contentLength = 0
boundary = networkString("%x%x" %
(int(time.time() * 1000000), os.getpid()))
if self.type:
contentType = self.type
else:
contentType = b'bytes' # It's what Apache does...
for start, end in byteRanges:
partOffset, partSize = self._rangeToOffsetAndSize(start, end)
if partOffset == partSize == 0:
continue
contentLength += partSize
matchingRangeFound = True
partContentRange = self._contentRange(partOffset, partSize)
partSeparator = networkString(
("\r\n"
"--%s\r\n"
"Content-type: %s\r\n"
"Content-range: %s\r\n"
"\r\n") % (nativeString(boundary), nativeString(contentType),
nativeString(partContentRange)))
contentLength += len(partSeparator)
rangeInfo.append((partSeparator, partOffset, partSize))
if not matchingRangeFound:
request.setResponseCode(http.REQUESTED_RANGE_NOT_SATISFIABLE)
request.setHeader(b'content-length', b'0')
request.setHeader(
b'content-range',
networkString('bytes */%d' % (self.getFileSize(), )))
return [], b''
finalBoundary = b"\r\n--" + boundary + b"--\r\n"
rangeInfo.append((finalBoundary, 0, 0))
request.setResponseCode(http.PARTIAL_CONTENT)
request.setHeader(
b'content-type',
networkString('multipart/byteranges; boundary="%s"' %
(nativeString(boundary), )))
request.setHeader(b'content-length',
intToBytes(contentLength + len(finalBoundary)))
return rangeInfo
def pickServer(self):
"""
Pick the next server.
This selects the next server from the list of SRV records according
to their priority and weight values, as set out by the default
algorithm specified in RFC 2782.
At the beginning of a round, L{servers} is populated with
L{orderedServers}, and the latter is made empty. L{servers}
is the list of candidates, and L{orderedServers} is the list of servers
that have already been tried.
First, all records are ordered by priority and weight in ascending
order. Then for each priority level, a running sum is calculated
over the sorted list of records for that priority. Then a random value
between 0 and the final sum is compared to each record in order. The
first record that is greater than or equal to that random value is
chosen and removed from the list of candidates for this round.
@return: A tuple of target hostname and port from the chosen DNS SRV
record.
@rtype: L{tuple} of native L{str} and L{int}
"""
assert self.servers is not None
assert self.orderedServers is not None
if not self.servers and not self.orderedServers:
# no SRV record, fall back..
return nativeString(self.domain), self.service
if not self.servers and self.orderedServers:
# start new round
self.servers = self.orderedServers
self.orderedServers = []
assert self.servers
self.servers.sort(key=lambda record: (record.priority, record.weight))
minPriority = self.servers[0].priority
index = 0
weightSum = 0
weightIndex = []
for x in self.servers:
if x.priority == minPriority:
weightSum += x.weight
weightIndex.append((index, weightSum))
index += 1
rand = random.randint(0, weightSum)
for index, weight in weightIndex:
if weight >= rand:
chosen = self.servers[index]
del self.servers[index]
self.orderedServers.append(chosen)
return str(chosen.target), chosen.port
raise RuntimeError("Impossible %s pickServer result." %
(self.__class__.__name__, ))
def render(self, request):
l = []
for k, v in sorted(list(request.received_cookies.items())):
l.append((nativeString(k), nativeString(v)))
l.sort()
return networkString(repr(l))