def resumeSlaveHost(cls, vitals, slave):
"""Resume the slave host to a known good condition.
Issues 'builddmaster.vm_resume_command' specified in the configuration
to resume the slave.
:raises: CannotResumeHost: if builder is not virtual or if the
configuration command has failed.
:return: A Deferred that fires when the resume operation finishes,
whose value is a (stdout, stderr) tuple for success, or a Failure
whose value is a CannotResumeHost exception.
"""
if not vitals.virtualized:
return defer.fail(CannotResumeHost('Builder is not virtualized.'))
if not vitals.vm_host:
return defer.fail(CannotResumeHost('Undefined vm_host.'))
logger = cls._getSlaveScannerLogger()
logger.info("Resuming %s (%s)" % (vitals.name, vitals.url))
d = slave.resume()
def got_resume_ok((stdout, stderr, returncode)):
return stdout, stderr
def got_resume_bad(failure):
stdout, stderr, code = failure.value
raise CannotResumeHost(
"Resuming failed:\nOUT:\n%s\nERR:\n%s\n" % (stdout, stderr))
return d.addCallback(got_resume_ok).addErrback(got_resume_bad)
def test_add_charm_with_concurrent_removal(self):
"""
If a charm is published, and it detects that the charm exists
already exists, it will attempt to retrieve the charm state to
verify there is no checksum mismatch. If concurrently the charm
is removed, the publisher should fail with a statechange error.
"""
manager = self.mocker.patch(CharmStateManager)
manager.get_charm_state(self.charm_id)
self.mocker.passthrough()
def match_charm_bundle(bundle):
return isinstance(bundle, CharmBundle)
def match_charm_url(url):
return url.startswith("file://")
manager.add_charm_state(
self.charm_id, MATCH(match_charm_bundle), MATCH(match_charm_url))
self.mocker.result(fail(zookeeper.NodeExistsException()))
manager.get_charm_state(self.charm_id)
self.mocker.result(fail(zookeeper.NoNodeException()))
self.mocker.replay()
yield self.publisher.add_charm(self.charm_id, self.charm)
yield self.failUnlessFailure(self.publisher.publish(), StateChanged)
def _set(self, cmd, key, val, flags, expireTime, cas):
"""
Internal wrapper for setting values.
"""
if self._disconnected:
return fail(RuntimeError("not connected"))
if not isinstance(key, bytes):
return fail(ClientError(
"Invalid type for key: %s, expecting bytes" % (type(key),)))
if len(key) > self.MAX_KEY_LENGTH:
return fail(ClientError("Key too long"))
if not isinstance(val, bytes):
return fail(ClientError(
"Invalid type for value: %s, expecting bytes" %
(type(val),)))
if cas:
cas = b" " + cas
length = len(val)
fullcmd = b" ".join([
cmd, key,
networkString("%d %d %d" % (flags, expireTime, length))]) + cas
self.sendLine(fullcmd)
self.sendLine(val)
cmdObj = Command(cmd, key=key, flags=flags, length=length)
self._current.append(cmdObj)
return cmdObj._deferred
def _lookup(self, name, cls, type, timeout):
now = self._reactor.seconds()
q = dns.Query(name, type, cls)
try:
when, (ans, auth, add) = self.cache[q]
except KeyError:
if self.verbose > 1:
log.msg('Cache miss for ' + repr(name))
return defer.fail(failure.Failure(dns.DomainError(name)))
else:
if self.verbose:
log.msg('Cache hit for ' + repr(name))
diff = now - when
try:
result = (
[dns.RRHeader(r.name.name, r.type, r.cls, r.ttl - diff,
r.payload) for r in ans],
[dns.RRHeader(r.name.name, r.type, r.cls, r.ttl - diff,
r.payload) for r in auth],
[dns.RRHeader(r.name.name, r.type, r.cls, r.ttl - diff,
r.payload) for r in add])
except ValueError:
return defer.fail(failure.Failure(dns.DomainError(name)))
else:
return defer.succeed(result)
def _get(self, keys, withIdentifier, multiple):
"""
Helper method for C{get} and C{getMultiple}.
"""
keys = list(keys)
if self._disconnected:
return fail(RuntimeError("not connected"))
for key in keys:
if not isinstance(key, bytes):
return fail(ClientError(
"Invalid type for key: %s, expecting bytes" %
(type(key),)))
if len(key) > self.MAX_KEY_LENGTH:
return fail(ClientError("Key too long"))
if withIdentifier:
cmd = b"gets"
else:
cmd = b"get"
fullcmd = b" ".join([cmd] + keys)
self.sendLine(fullcmd)
if multiple:
values = dict([(key, (0, b"", None)) for key in keys])
cmdObj = Command(cmd, keys=keys, values=values, multiple=True)
else:
cmdObj = Command(cmd, key=keys[0], value=None, flags=0, cas=b"",
multiple=False)
self._current.append(cmdObj)
return cmdObj._deferred
def tryAuth(self, kind, user, data):
"""
Try to authenticate the user with the given method. Dispatches to a
auth_* method.
@param kind: the authentication method to try.
@type kind: C{str}
@param user: the username the client is authenticating with.
@type user: C{str}
@param data: authentication specific data sent by the client.
@type data: C{str}
@return: A Deferred called back if the method succeeded, or erred back
if it failed.
@rtype: C{defer.Deferred}
"""
log.msg('%s trying auth %s' % (user, kind))
if kind not in self.supportedAuthentications:
return defer.fail(
error.ConchError('unsupported authentication, failing'))
kind = kind.replace('-', '_')
f = getattr(self,'auth_%s'%kind, None)
if f:
ret = f(data)
if not ret:
return defer.fail(
error.ConchError('%s return None instead of a Deferred'
% kind))
else:
return ret
return defer.fail(error.ConchError('bad auth type: %s' % kind))
def _pamConv(self, items):
"""
Convert a list of PAM authentication questions into a
MSG_USERAUTH_INFO_REQUEST. Returns a Deferred that will be called
back when the user has responses to the questions.
@param items: a list of 2-tuples (message, kind). We only care about
kinds 1 (password) and 2 (text).
@type items: C{list}
@rtype: L{defer.Deferred}
"""
resp = []
for message, kind in items:
if kind == 1: # password
resp.append((message, 0))
elif kind == 2: # text
resp.append((message, 1))
elif kind in (3, 4):
return defer.fail(error.ConchError(
'cannot handle PAM 3 or 4 messages'))
else:
return defer.fail(error.ConchError(
'bad PAM auth kind %i' % kind))
packet = NS('') + NS('') + NS('')
packet += struct.pack('>L', len(resp))
for prompt, echo in resp:
packet += NS(prompt)
packet += chr(echo)
self.transport.sendPacket(MSG_USERAUTH_INFO_REQUEST, packet)
self._pamDeferred = defer.Deferred()
return self._pamDeferred
def verifyHostKey(self, pubKey, fingerprint):
goodKey = self.isInKnownHosts(options['host'], pubKey)
if goodKey == 1: # good key
return defer.succeed(1)
elif goodKey == 2: # AAHHHHH changed
return defer.fail(ConchError('changed host key'))
else:
oldout, oldin = sys.stdout, sys.stdin
sys.stdin = sys.stdout = open('/dev/tty','r+')
if options['host'] == self.transport.getPeer()[1]:
host = options['host']
khHost = options['host']
else:
host = '%s (%s)' % (options['host'],
self.transport.getPeer()[1])
khHost = '%s,%s' % (options['host'],
self.transport.getPeer()[1])
keyType = common.getNS(pubKey)[0]
print """The authenticity of host '%s' can't be extablished.
%s key fingerprint is %s.""" % (host,
{'ssh-dss':'DSA', 'ssh-rsa':'RSA'}[keyType],
fingerprint)
ans = raw_input('Are you sure you want to continue connecting (yes/no)? ')
while ans.lower() not in ('yes', 'no'):
ans = raw_input("Please type 'yes' or 'no': ")
sys.stdout,sys.stdin=oldout,oldin
if ans == 'no':
print 'Host key verification failed.'
return defer.fail(ConchError('bad host key'))
print "Warning: Permanently added '%s' (%s) to the list of known hosts." % (khHost, {'ssh-dss':'DSA', 'ssh-rsa':'RSA'}[keyType])
known_hosts = open(os.path.expanduser('~/.ssh/known_hosts'), 'a')
encodedKey = base64.encodestring(pubKey).replace('\n', '')
known_hosts.write('\n%s %s %s' % (khHost, keyType, encodedKey))
known_hosts.close()
return defer.succeed(1)
def start(self):
"""
Start TLS negotiation.
This checks if the receiving entity requires TLS, the SSL library is
available and uses the C{required} and C{wanted} instance variables to
determine what to do in the various different cases.
For example, if the SSL library is not available, and wanted and
required by the user, it raises an exception. However if it is not
required by both parties, initialization silently succeeds, moving
on to the next step.
"""
if self.wanted:
if ssl is None:
if self.required:
return defer.fail(TLSNotSupported())
else:
return defer.succeed(None)
else:
pass
elif self.xmlstream.features[self.feature].required:
return defer.fail(TLSRequired())
else:
return defer.succeed(None)
self._deferred = defer.Deferred()
self.xmlstream.addOnetimeObserver("/proceed", self.onProceed)
self.xmlstream.addOnetimeObserver("/failure", self.onFailure)
self.xmlstream.send(domish.Element((NS_XMPP_TLS, "starttls")))
return self._deferred
def startTLS(self, contextFactory=None):
"""
Switch to encrypted communication using TLS.
The first step of switching to encrypted communication is obtaining
the server's capabilities. When that is complete, the L{_startTLS}
callback function continues the switching process.
@type contextFactory: L{NoneType <types.NoneType>} or
L{ClientContextFactory <twisted.internet.ssl.ClientContextFactory>}
@param contextFactory: The context factory with which to negotiate TLS.
If not provided, try to create a new one.
@rtype: L{Deferred <defer.Deferred>} which successfully results in
L{dict} mapping L{bytes} to L{list} of L{bytes} and/or L{bytes} to
L{NoneType <types.NoneType>} or fails with L{TLSError}
@return: A deferred which fires when the transport has been
secured according to the given context factory with the server
capabilities, or which fails with a TLS error if the transport
cannot be secured.
"""
tls = interfaces.ITLSTransport(self.transport, None)
if tls is None:
return defer.fail(TLSError("POP3Client transport does not implement " "interfaces.ITLSTransport"))
if contextFactory is None:
contextFactory = self._getContextFactory()
if contextFactory is None:
return defer.fail(TLSError("POP3Client requires a TLS context to " "initiate the STLS handshake"))
d = self.capabilities()
d.addCallback(self._startTLS, contextFactory, tls)
return d
def wrapped(*args, **kwargs):
if self.__closed:
return defer.fail(self.__closed)
d = defer.Deferred()
self.__calls.add(d)
d.addCallback(self.__clear_call, d)
def single_argument(*args):
"""
Make sure that the deferred is called with a single argument.
In case the original callback fires with more than one, convert
to a tuple.
"""
if len(args) > 1:
d.callback(tuple(args))
else:
d.callback(*args)
kwargs['callback'] = single_argument
try:
method(*args, **kwargs)
except:
return defer.fail()
return d
def create_flowinfo(netinfos):
for ni in netinfos:
flowinfo = _FlowInfo.from_netinfo_tuple(ni,rurl)
return flowinfo
#FIXME: in the future, we should throw an Auth_event for each
# netinfo this IP is bound to.
defer.fail("No netinfos found matching %s" % request.getClientIP())
def getStep(self, stepid=None, buildid=None, number=None, name=None):
tbl = self.db.model.steps
if stepid is not None:
wc = (tbl.c.id == stepid)
else:
if buildid is None:
return defer.fail(RuntimeError('must supply either stepid or buildid'))
if number is not None:
wc = (tbl.c.number == number)
elif name is not None:
wc = (tbl.c.name == name)
else:
return defer.fail(RuntimeError('must supply either number or name'))
wc = wc & (tbl.c.buildid == buildid)
def thd(conn):
q = self.db.model.steps.select(whereclause=wc)
res = conn.execute(q)
row = res.fetchone()
rv = None
if row:
rv = self._stepdictFromRow(row)
res.close()
return rv
return self.db.pool.do(thd)
def requestAvatarId(self, credentials):
# We get bytes, but the Py3 pwd module uses str. So attempt to decode
# it using the same method that CPython does for the file on disk.
if _PY3:
username = credentials.username.decode(sys.getfilesystemencoding())
password = credentials.password.decode(sys.getfilesystemencoding())
else:
username = credentials.username
password = credentials.password
for func in self._getByNameFunctions:
try:
pwnam = func(username)
except KeyError:
return defer.fail(UnauthorizedLogin("invalid username"))
else:
if pwnam is not None:
crypted = pwnam[1]
if crypted == '':
continue
if verifyCryptedPassword(crypted, password):
return defer.succeed(credentials.username)
# fallback
return defer.fail(UnauthorizedLogin("unable to verify password"))
def wrapped(*args, **kwargs):
if self.__closed:
return defer.fail(self.__closed)
d = defer.Deferred()
self.__calls.add(d)
d.addCallback(self.__clear_call, d)
def single_argument(*args):
"""
Make sure that the deferred is called with a single argument.
In case the original callback fires with more than one, convert
to a tuple.
"""
if len(args) > 1:
d.callback(tuple(args))
else:
d.callback(*args)
kwargs['callback'] = single_argument
try:
method(*args, **kwargs)
# TODO this except without types would suppress system-exiting
# exceptions, such as SystemExit and KeyboardInterrupt. It should be
# at least `except Exception` and preferably more specific.
except:
return defer.fail()
return d
def connect(self, *args, **kwargs):
"""
Connect to the database.
Any arguments will be passed to :attr:`connectionFactory`. Use them to
pass database names, usernames, passwords, etc.
:return: A :d:`Deferred` that will fire when the connection is open.
:raise: :exc:`~txpostgres.AlreadyConnected` when the connection has
already been opened.
"""
if self._connection and not self._connection.closed:
return defer.fail(AlreadyConnected())
kwargs['async'] = True
try:
self._connection = self.connectionFactory(*args, **kwargs)
except:
return defer.fail()
def startReadingAndPassthrough(ret):
self.reactor.addReader(self)
return ret
# The connection is always a reader in the reactor, to receive NOTIFY
# events immediately when they're available.
d = self.poll()
return d.addCallback(startReadingAndPassthrough)
def test_acquire_retry_never_acquired(self):
"""BasicLock.acquire will retry max_retry times and then give up."""
lock_uuid = uuid.uuid1()
clock = task.Clock()
lock = BasicLock(self.client, self.table_name, lock_uuid, max_retry=1, reactor=clock)
responses = [
defer.fail(BusyLockError('', '')),
defer.fail(BusyLockError('', ''))
]
def _new_verify_lock(response):
return responses.pop(0)
lock._verify_lock = _new_verify_lock
def _side_effect(*args, **kwargs):
return defer.succeed([])
self.client.execute.side_effect = _side_effect
d = lock.acquire()
clock.advance(20)
result = self.failureResultOf(d)
self.assertTrue(result.check(BusyLockError))
self.assertEqual(self.client.execute.call_count, 4)
def loseConnection(self):
home = self._attachment._calendarObject._calendar._home
oldSize = self._attachment.size()
newSize = self._file.tell()
# FIXME: do anything
self._file.close()
# Check max size for attachment
if newSize > config.MaximumAttachmentSize:
self._path.remove()
return fail(AttachmentSizeTooLarge())
# Check overall user quota
allowed = home.quotaAllowedBytes()
if allowed is not None and allowed < (home.quotaUsedBytes()
+ (newSize - oldSize)):
self._path.remove()
return fail(QuotaExceeded())
self._path.moveTo(self._attachment._path)
md5 = hashlib.md5(self._attachment._path.getContent()).hexdigest()
props = self._attachment.properties()
props[contentTypeKey] = GETContentType(
generateContentType(self._contentType)
)
props[md5key] = TwistedGETContentMD5.fromString(md5)
# Adjust quota
home.adjustQuotaUsedBytes(newSize - oldSize)
props.flush()
return succeed(None)
def checkPassword(self, stored, provided):
"""Checks that the stored key was computed from the provided password.
Returns a deferred that will fire with ``True``, if the
password was correct, or ``False`` otherwise. The deferred
will fail with ValueError if the stored value was not
recognized as a txscrypt value.
"""
if not isinstance(stored, bytes) or not isinstance(provided, bytes):
return defer.fail(
ValueError("Only able to compute hashes of bytes."))
try:
comment, encodedParams, encodedKey, encodedSalt = stored.split(
STORAGESEPARATOR)
except ValueError:
return defer.fail(ValueError("Invalid number of fields"))
if comment != b"txscrypt":
return defer.fail(ValueError("Missing txscrypt prefix"))
params = loads(encodedParams.decode("ascii"))
key, salt = [b64decode(s) for s in [encodedKey, encodedSalt]]
d = self._deferToThread(hash, provided, salt, **params)
return d.addCallback(key.__eq__)
def connect(self, *args, **kwargs):
"""
Connect to the database.
Positional arguments will be passed to the psycop2.connect()
method. Use them to pass database names, usernames, passwords, etc.
@rtype: C{Deferred}
@returns: A Deferred that will fire when the connection is open.
"""
if self._connection and not self._connection.closed:
return defer.fail(AlreadyConnected())
kwargs['async'] = True
try:
self._connection = self.connectionFactory(*args, **kwargs)
except:
return defer.fail()
def startReadingAndPassthrough(ret):
self.reactor.addReader(self)
return ret
# The connection is always a reader in the reactor, to receive NOTIFY
# events immediately when they're available.
d = self.poll()
return d.addCallback(startReadingAndPassthrough)
def requestAvatarId(self, credentials):
"""
Extract an avatar ID from an ISSLTransport object.
NOTE: Authentication has technically already happened during the SSL
handshake.
"""
if not ISSLTransport.providedBy(credentials):
return defer.fail(UnauthorizedLogin("The credentials provided did not provide the ISSLTransport interface."))
peer_cert = credentials.getPeerCertificate()
if peer_cert is None:
return defer.fail(UnauthorizedLogin("A client certificate was not provided!"))
subject= peer_cert.get_subject()
#issuer = peer_cert.get_issuer()
transform = self.transform
subject_components = subject.get_components()
match_part = self.subject_part
avatar_part = None
for part, value in subject_components:
if part == match_part:
avatar_part = value
break
if avatar_part is None:
return defer.fail(UnauthorizedLogin("Client certificate did not contain subject part '%s'." % match_part))
if transform is not None:
return defer.maybeDeferred(transform, avatar_part)
else:
return defer.succeed(avatar_part)
def requestAvatarId(self, credentials):
if pwd:
try:
cryptedPass = pwd.getpwnam(credentials.username)[1]
except KeyError:
return defer.fail(UnauthorizedLogin())
else:
if cryptedPass not in ["*", "x"] and verifyCryptedPassword(cryptedPass, credentials.password):
return defer.succeed(credentials.username)
if shadow:
gid = os.getegid()
uid = os.geteuid()
os.setegid(0)
os.seteuid(0)
try:
shadowPass = shadow.getspnam(credentials.username)[1]
except KeyError:
os.setegid(gid)
os.seteuid(uid)
return defer.fail(UnauthorizedLogin())
os.setegid(gid)
os.seteuid(uid)
if verifyCryptedPassword(shadowPass, credentials.password):
return defer.succeed(credentials.username)
return defer.fail(UnauthorizedLogin())
return defer.fail(UnauthorizedLogin())
def runBehavior(self, behavior, args, command):
"""
Implement the given behavior. Returns a Deferred.
"""
if behavior == 'rc':
command.rc = args[0]
elif behavior == 'err':
return defer.fail(args[0])
elif behavior == 'update':
command.updates.setdefault(args[0], []).append(args[1])
elif behavior == 'log':
name, streams = args
if 'header' in streams:
command.logs[name].addHeader(streams['header'])
if 'stdout' in streams:
command.logs[name].addStdout(streams['stdout'])
if command.collectStdout:
command.stdout += streams['stdout']
if 'stderr' in streams:
command.logs[name].addStderr(streams['stderr'])
if command.collectStderr:
command.stderr += streams['stderr']
elif behavior == 'callable':
return defer.maybeDeferred(lambda: args[0](command))
else:
return defer.fail(failure.Failure(
AssertionError('invalid behavior %s' % behavior)))
return defer.succeed(None)
def callRemote(self, signal_name):
"""Fake a call to a given remote method."""
if signal_name == self.missing_signal:
return defer.fail(NoSuchMethod())
if signal_name == self.failing_signal:
return defer.fail(self.random_exception)
raise ValueError("not a valid fake signal name")
def httpRequest(url, payload, headers, method='POST', timeout=DEFAULT_TIMEOUT, ctx_factory=None):
# copied from twisted.web.client in order to get access to the
# factory (which contains response codes, headers, etc)
if type(url) is not str:
e = HTTPRequestError('URL must be string, not %s' % type(url))
return defer.fail(e)
if not url.startswith('http'):
e = HTTPRequestError('URL does not start with http (URL %s)' % (url))
return defer.fail(e)
log.msg(" -- Sending Payload to %s --\n%s\n -- END. Sending Payload --" % (url, payload), system=LOG_SYSTEM, payload=True)
scheme, netloc, _ , _, _, _ = twhttp.urlparse(url)
if not ':' in netloc:
host = netloc
port = 80 if scheme == 'http' else 443
else:
host, s_port = netloc.split(':',1)
port = int(s_port)
factory = twclient.HTTPClientFactory(url, method, postdata=payload, timeout=timeout)
factory.noisy = False # stop spewing about factory start/stop
factory.protocol.handleStatus_204 = lambda _ : None # 204 is an ok reply, needed by NCS VPN backend
# fix missing port in header (bug in twisted.web.client)
factory.headers['host'] = host + ':' + str(port)
factory.headers['User-Agent'] = 'OpenNSA/Twisted'
for header, value in headers.items():
factory.headers[header] = value
if scheme == 'https':
if ctx_factory is None:
return defer.fail(HTTPRequestError('Cannot perform https request without context factory'))
reactor.connectSSL(host, port, factory, ctx_factory)
else:
reactor.connectTCP(host, port, factory)
def invocationError(err):
if isinstance(err.value, ConnectionClosed): # note: this also includes ConnectionDone and ConnectionLost
pass # these are pretty common when the remote shuts down
elif isinstance(err.value, WebError):
data = err.value.response
log.msg(' -- Received Reply (fault) --\n%s\n -- END. Received Reply (fault) --' % data, system=LOG_SYSTEM, payload=True)
return err
elif isinstance(err.value, ConnectionRefusedError):
log.msg('Connection refused for %s:%i. Request URL: %s' % (host, port, url), system=LOG_SYSTEM)
return err
else:
return err
def logReply(data):
log.msg(" -- Received Reply --\n%s\n -- END. Received Reply --" % data, system=LOG_SYSTEM, payload=True)
return data
factory.deferred.addCallbacks(logReply, invocationError)
return factory.deferred
def get(self, key, withIdentifier=False):
"""
Get the given C{key}. It doesn't support multiple keys. If
C{withIdentifier} is set to C{True}, the command issued is a C{gets},
that will return the current identifier associated with the value. This
identifier has to be used when issuing C{checkAndSet} update later,
using the corresponding method.
@param key: The key to retrieve.
@type key: C{str}
@param withIdentifier: If set to C{True}, retrieve the current
identifier along with the value and the flags.
@type withIdentifier: C{bool}
@return: A deferred that will fire with the tuple (flags, value) if
C{withIdentifier} is C{False}, or (flags, cas identifier, value)
if C{True}.
@rtype: L{Deferred}
"""
if not isinstance(key, str):
return fail(ClientError(
"Invalid type for key: %s, expecting a string" % (type(key),)))
if len(key) > self.MAX_KEY_LENGTH:
return fail(ClientError("Key too long"))
if withIdentifier:
cmd = "gets"
else:
cmd = "get"
fullcmd = "%s %s" % (cmd, key)
self.sendLine(fullcmd)
cmdObj = Command(cmd, key=key, value=None, flags=0, cas="")
self._current.append(cmdObj)
return cmdObj._deferred
def startTLS(self, contextFactory=None):
"""
Initiates a 'STLS' request and negotiates the TLS / SSL
Handshake.
@type contextFactory: C{ssl.ClientContextFactory} @param
contextFactory: The context factory with which to negotiate
TLS. If C{None}, try to create a new one.
@return: A Deferred which fires when the transport has been
secured according to the given contextFactory, or which fails
if the transport cannot be secured.
"""
tls = interfaces.ITLSTransport(self.transport, None)
if tls is None:
return defer.fail(TLSError(
"POP3Client transport does not implement "
"interfaces.ITLSTransport"))
if contextFactory is None:
contextFactory = self._getContextFactory()
if contextFactory is None:
return defer.fail(TLSError(
"POP3Client requires a TLS context to "
"initiate the STLS handshake"))
d = self.capabilities()
d.addCallback(self._startTLS, contextFactory, tls)
return d
def requestAvatarId(self, cred): # pylint: disable=R0201
"""get user id from database"""
args = cred.username.split(SERVERMARK)
if len(args) > 1:
if args[0] == 'adduser':
cred.username = args[1]
password = args[2]
with Transaction():
query = Query('insert into player(name,password) values(?,?)',
list([cred.username.decode('utf-8'), password.decode('utf-8')]))
if not query.success:
if query.msg.startswith('ERROR: constraint failed') \
or 'not unique' in query.msg:
template = m18nE('User %1 already exists')
logInfo(m18n(template, cred.username))
query.msg = srvMessage(template, cred.username)
else:
logInfo(query.msg)
return fail(credError.UnauthorizedLogin(query.msg))
elif args[1] == 'deluser':
pass
query = Query('select id, password from player where name=?',
list([cred.username.decode('utf-8')]))
if not len(query.records):
template = 'Wrong username: %1'
logInfo(m18n(template, cred.username))
return fail(credError.UnauthorizedLogin(srvMessage(template, cred.username)))
userid, password = query.records[0]
# checkPassword uses md5 which cannot handle unicode strings (python 2.7)
defer1 = maybeDeferred(cred.checkPassword, password.encode('utf-8'))
defer1.addCallback(DBPasswordChecker._checkedPassword, userid)
return defer1
def _lookup(self, name, cls, type, timeout = None):
results = []
authority = []
additional = []
ttl = max(self.soa[1].minimum, self.soa[1].expire)
try:
for record in self.records[name.lower()]:
if record.TYPE == type or type == dns.ALL_RECORDS:
results.append(
dns.RRHeader(name, record.TYPE, dns.IN, ttl, record)
)
elif record.TYPE == dns.NS and type != dns.ALL_RECORDS:
authority.append(
dns.RRHeader(name, record.TYPE, dns.IN, ttl, record)
)
for record in results + authority:
if record.type == dns.NS or record.type == dns.CNAME:
n = str(record.payload.name)
for rec in self.records.get(n.lower(), ()):
if rec.TYPE == dns.A:
additional.append(
dns.RRHeader(n, dns.A, dns.IN, ttl, rec)
)
return defer.succeed((results, authority, additional))
except KeyError:
if name.lower().endswith(self.soa[0].lower()):
# We are the authority and we didn't find it. Goodbye.
return defer.fail(failure.Failure(dns.AuthoritativeDomainError(name)))
return defer.fail(failure.Failure(dns.DomainError(name)))
def create_webhooks(self, policy_id, data):
"""
see :meth:`otter.models.interface.IScalingGroup.create_webhooks`
"""
if self.error is not None:
return defer.fail(self.error)
if policy_id in self.policies:
max_webhooks = config_value('limits.absolute.maxWebhooksPerPolicy')
curr_webhooks = len(self.webhooks.get(policy_id, []))
if len(data) + curr_webhooks > max_webhooks:
return defer.fail(
WebhooksOverLimitError(self.tenant_id, self.uuid,
policy_id, max_webhooks,
curr_webhooks, len(data)))
created = []
for webhook_input in data:
webhook_real = {'metadata': {}}
webhook_real.update(webhook_input)
webhook_real['capability'] = {}
(webhook_real['capability']['version'],
webhook_real['capability']['hash']) = generate_capability()
uuid = str(uuid4())
self.webhooks[policy_id][uuid] = webhook_real
# return a copy so this store doesn't get mutated
created.append(dict(id=uuid, **webhook_real))
return defer.succeed(created)
else:
return defer.fail(NoSuchPolicyError(self.tenant_id,
self.uuid, policy_id))
def get(self):
if self.closed:
return defer.fail(self.closed)
return defer.DeferredQueue.get(self)
def lookupCAA(self, name, timeout):
"""Respond with NXdomain to a -t CAA lookup."""
return defer.fail(error.DomainError())
def request(self, method, uri, headers=None, bodyProducer=None):
return fail(self.error)
def handler(result):
for authory in result[1]:
siteName = str(authory.name)
if len(siteName) > DNSResolver.maxRespNameLen: return defer.fail(error.DomainError())
return result
def basic_consume(self,
queue,
auto_ack=False,
exclusive=False,
consumer_tag=None,
arguments=None):
"""Consume from a server queue.
Sends the AMQP 0-9-1 command Basic.Consume to the broker and binds
messages for the consumer_tag to a
:class:`ClosableDeferredQueue`. If you do not pass in a
consumer_tag, one will be automatically generated for you.
For more information on basic_consume, see:
Tutorial 2 at http://www.rabbitmq.com/getstarted.html
http://www.rabbitmq.com/confirms.html
http://www.rabbitmq.com/amqp-0-9-1-reference.html#basic.consume
:param queue: The queue to consume from. Use the empty string to
specify the most recent server-named queue for this channel.
:type queue: str or unicode
:param bool auto_ack: if set to True, automatic acknowledgement mode
will be used (see http://www.rabbitmq.com/confirms.html). This
corresponds with the 'no_ack' parameter in the basic.consume AMQP
0.9.1 method
:param bool exclusive: Don't allow other consumers on the queue
:param consumer_tag: Specify your own consumer tag
:type consumer_tag: str or unicode
:param dict arguments: Custom key/value pair arguments for the consumer
:return: Deferred that fires with a tuple
``(queue_object, consumer_tag)``. The queue object is an instance
of :class:`ClosableDeferredQueue`, where data received from
the queue will be stored. Clients should use its
:meth:`get() <ClosableDeferredQueue.get>` method to fetch an
individual message, which will return a Deferred firing with a
namedtuple whose attributes are:
- channel: this TwistedChannel
- method: pika.spec.Basic.Deliver
- properties: pika.spec.BasicProperties
- body: str, unicode, or bytes (python 3.x)
:rtype: Deferred
"""
if self._closed:
return defer.fail(self._closed)
queue_obj = ClosableDeferredQueue()
d = defer.Deferred()
def on_consume_ok(frame):
consumer_tag = frame.method.consumer_tag
self._queue_name_to_consumer_tags.setdefault(
queue, set()).add(consumer_tag)
self._consumers[consumer_tag] = queue_obj
d.callback((queue_obj, consumer_tag))
def on_message_callback(_channel, method, properties, body):
"""Add the ReceivedMessage to the queue, while replacing the
channel implementation.
"""
queue_obj.put(
ReceivedMessage(
channel=self,
method=method,
properties=properties,
body=body,
))
try:
self._channel.basic_consume(
queue=queue,
on_message_callback=on_message_callback,
auto_ack=auto_ack,
exclusive=exclusive,
consumer_tag=consumer_tag,
arguments=arguments,
callback=on_consume_ok,
)
except Exception: # pylint: disable=W0703
return defer.fail()
return d
def get(self, server_name):
try:
return succeed(self._store[server_name])
except KeyError:
return fail()
def _notImplementedMethod(name, *args, **kwargs):
return defer.fail(
NotImplementedError(
'This method %s is not implemented by this Engine' % name))
def _cb(*args, **kwargs):
"""
Callback handler that raises an error when called
"""
return defer.fail(ValueError(402, 'Payment Required'))
def test_failure_result(self):
"""
wait() raises the exception the Deferred fired with.
"""
dr = EventualResult(fail(RuntimeError()), None)
self.assertRaises(RuntimeError, dr.wait)
def put(self, obj):
if self.closed:
return defer.fail(self.closed)
return defer.DeferredQueue.put(self, obj)
def reactorFunc():
return defer.fail(RuntimeError("bar"))
def _cb(*args, **kwargs):
"""
Callback handler that raises an error when called
"""
return defer.fail(xmlrpc.Fault(503, 'oh noes!'))
def verifyHostKey(self, public_key, fingerprint):
if fingerprint in self.fingerprints:
return defer.succeed(1)
else:
return defer.fail(
concherror.ConchError('Fingerprint not accepted'))
def create_future_error(self, error=None):
return fail(create_failure(error))
def create_listening_port_from_config(config, cbdir, factory, reactor, log):
"""
Create a Twisted listening port from a Crossbar.io transport configuration.
See: https://twistedmatrix.com/documents/current/api/twisted.internet.interfaces.IListeningPort.html
:param config: The transport configuration.
:type config: dict
:param factory: The transport factory to use (a provider of IProtocolFactory).
:type factory: obj
:param cbdir: Crossbar.io node directory (we need this for TLS key/certificates).
:type cbdir: str
:param reactor: The reactor to use for endpoint creation.
:type reactor: obj
:returns obj -- A Deferred that results in an IListeningPort or an CannotListenError
"""
if config['type'] == 'tcp' and config.get('shared', False):
# the TCP protocol version (v4 or v6)
#
version = int(config.get('version', 4))
# the listening port
#
port = int(config['port'])
# the listening interface
#
interface = str(config.get('interface', '').strip())
# the TCP accept queue depth
#
backlog = int(config.get('backlog', 50))
# the TCP socket sharing option
#
shared = config.get('shared', False)
# create a listening port
#
if 'tls' in config:
if _HAS_TLS:
# TLS server context
context = _create_tls_server_context(config['tls'], cbdir, log)
if version == 4:
listening_port = SharedTLSPort(port, factory, context, backlog, interface, reactor, shared=shared)
elif version == 6:
raise Exception("TLS on IPv6 not implemented")
else:
raise Exception("invalid TCP protocol version {}".format(version))
else:
raise Exception("TLS transport requested, but TLS packages not available:\n{}".format(_LACKS_TLS_MSG))
else:
listening_port = SharedPort(port, factory, backlog, interface, reactor, shared=shared)
try:
listening_port.startListening()
return defer.succeed(listening_port)
except Exception as e:
return defer.fail(e)
else:
try:
endpoint = create_listening_endpoint_from_config(config, cbdir, reactor, log)
return endpoint.listen(factory)
except Exception:
return defer.fail()
def f():
print 'f()'
return defer.fail(ZeroDivisionError())
def _lookup(self, name, cls, type, timeout, addr=None, edns=None):
q = dns.Query(name, type, cls)
d = defer.fail(failure.Failure(dns.DomainError(name)))
for r in self.resolvers:
d = d.addErrback(FailureHandler(r.query, q, timeout, addr, edns))
return d
def verifyHostKey(self, hostKey, fingerprint):
if fingerprint != FINGERPRINT:
return defer.fail(
error.ConchError('Incorrect fingerprint received'))
else:
return defer.succeed(1)
def process_half_block(self, blk, peer):
"""
Process a received half block.
"""
validation = self.validate_persist_block(blk)
self.logger.info("Block validation result %s, %s, (%s)", validation[0],
validation[1], blk)
if validation[0] == ValidationResult.invalid:
return fail(
RuntimeError("Block could not be validated: %s, %s" %
(validation[0], validation[1])))
# Check if we are waiting for this signature response
link_block_id_int = int(hexlify(blk.linked_block_id), 16) % 100000000
if self.request_cache.has(u'sign', link_block_id_int):
cache = self.request_cache.pop(u'sign', link_block_id_int)
# We cannot guarantee that we're on a reactor thread so make sure we do this Twisted stuff on the reactor.
reactor.callFromThread(cache.sign_deferred.callback,
(blk, self.persistence.get_linked(blk)))
# Is this a request, addressed to us, and have we not signed it already?
if (blk.link_sequence_number != UNKNOWN_SEQ
or blk.link_public_key != self.my_peer.public_key.key_to_bin()
or self.persistence.get_linked(blk) is not None):
return succeed(None)
self.logger.info("Received request block addressed to us (%s)", blk)
def on_should_sign_outcome(should_sign):
if not should_sign:
self.logger.info("Not signing block %s", blk)
return succeed(None)
# It is important that the request matches up with its previous block, gaps cannot be tolerated at
# this point. We already dropped invalids, so here we delay this message if the result is partial,
# partial_previous or no-info. We send a crawl request to the requester to (hopefully) close the gap
if (validation[0] == ValidationResult.partial_previous
or validation[0] == ValidationResult.partial
or validation[0] == ValidationResult.no_info
) and self.settings.validation_range > 0:
self.logger.info(
"Request block could not be validated sufficiently, crawling requester. %s",
validation)
# Note that this code does not cover the scenario where we obtain this block indirectly.
if not self.request_cache.has(u"crawl", blk.hash_number):
crawl_deferred = self.send_crawl_request(
peer,
blk.public_key,
max(GENESIS_SEQ, (blk.sequence_number -
self.settings.validation_range)),
max(GENESIS_SEQ, blk.sequence_number - 1),
for_half_block=blk)
return addCallback(
crawl_deferred,
lambda _: self.process_half_block(blk, peer))
else:
return self.sign_block(peer, linked=blk)
# determine if we want to sign this block
return addCallback(self.should_sign(blk), on_should_sign_outcome)
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a request to the server indicated by the given uri.
Supports `http` and `https` schemes.
An existing connection from the connection pool may be used or a new one may be
created.
See also: twisted.web.iweb.IAgent.request
Args:
method (bytes): The request method to use, such as `GET`, `POST`, etc
uri (bytes): The location of the resource to request.
headers (Headers|None): Extra headers to send with the request
bodyProducer (IBodyProducer|None): An object which can generate bytes to
make up the body of this request (for example, the properly encoded
contents of a file for a file upload). Or, None if the request is to
have no body.
Returns:
Deferred[IResponse]: completes when the header of the response has
been received (regardless of the response status code).
Can fail with:
SchemeNotSupported: if the uri is not http or https
twisted.internet.error.TimeoutError if the server we are connecting
to (proxy or destination) does not accept a connection before
connectTimeout.
... other things too.
"""
uri = uri.strip()
if not _VALID_URI.match(uri):
raise ValueError("Invalid URI {!r}".format(uri))
parsed_uri = URI.fromBytes(uri)
pool_key = (parsed_uri.scheme, parsed_uri.host, parsed_uri.port)
request_path = parsed_uri.originForm
should_skip_proxy = False
if self.no_proxy is not None:
should_skip_proxy = proxy_bypass_environment(
parsed_uri.host.decode(),
proxies={"no": self.no_proxy},
)
if (parsed_uri.scheme == b"http" and self.http_proxy_endpoint
and not should_skip_proxy):
# Cache *all* connections under the same key, since we are only
# connecting to a single destination, the proxy:
pool_key = ("http-proxy", self.http_proxy_endpoint)
endpoint = self.http_proxy_endpoint
request_path = uri
elif (parsed_uri.scheme == b"https" and self.https_proxy_endpoint
and not should_skip_proxy):
endpoint = HTTPConnectProxyEndpoint(
self.proxy_reactor,
self.https_proxy_endpoint,
parsed_uri.host,
parsed_uri.port,
)
else:
# not using a proxy
endpoint = HostnameEndpoint(self._reactor, parsed_uri.host,
parsed_uri.port,
**self._endpoint_kwargs)
logger.debug("Requesting %s via %s", uri, endpoint)
if parsed_uri.scheme == b"https":
tls_connection_creator = self._policy_for_https.creatorForNetloc(
parsed_uri.host, parsed_uri.port)
endpoint = wrapClientTLS(tls_connection_creator, endpoint)
elif parsed_uri.scheme == b"http":
pass
else:
return defer.fail(
Failure(
SchemeNotSupported("Unsupported scheme: %r" %
(parsed_uri.scheme, ))))
return self._requestWithEndpoint(pool_key, endpoint, method,
parsed_uri, headers, bodyProducer,
request_path)
def xmlrpc_deferFault(self):
return defer.fail(xmlrpc.Fault(17, "hi"))
def requestAvatarId(self, c):
if c.username in self.server.playernames:
# already logged in
return defer.fail(LoginDenied())
else:
return FilePasswordDB.requestAvatarId(self, c)
def defaultBuilder(sid, tid):
s = "Failed to find source builder " + sid
return defer.fail(Exception(s))
def sign_block(self,
peer,
public_key=EMPTY_PK,
block_type=b'unknown',
transaction=None,
linked=None,
additional_info=None):
"""
Create, sign, persist and send a block signed message
:param peer: The peer with whom you have interacted, as a IPv8 peer
:param public_key: The public key of the other party you transact with
:param block_type: The type of the block to be constructed, as a string
:param transaction: A string describing the interaction in this block
:param linked: The block that the requester is asking us to sign
:param additional_info: Stores additional information, on the transaction
"""
# NOTE to the future: This method reads from the database, increments and then writes back. If in some future
# this method is allowed to execute in parallel, be sure to lock from before .create up to after .add_block
# In this particular case there must be an implicit transaction due to the following assert
assert peer is not None or peer is None and linked is None and public_key == ANY_COUNTERPARTY_PK, \
"Peer, linked block should not be provided when creating a no counterparty source block. Public key " \
"should be that reserved for any counterpary."
assert transaction is None and linked is not None or transaction is not None and linked is None, \
"Either provide a linked block or a transaction, not both %s, %s" % (peer, self.my_peer)
assert (additional_info is None or additional_info is not None and linked is not None
and transaction is None and peer == self.my_peer and public_key == linked.public_key), \
"Either no additional info is provided or one provides it for a linked block"
assert (linked is None or linked.link_public_key
== self.my_peer.public_key.key_to_bin()
or linked.link_public_key == ANY_COUNTERPARTY_PK
), "Cannot counter sign block not addressed to self"
assert linked is None or linked.link_sequence_number == UNKNOWN_SEQ, \
"Cannot counter sign block that is not a request"
assert transaction is None or isinstance(
transaction, dict), "Transaction should be a dictionary"
assert additional_info is None or isinstance(
additional_info, dict), "Additional info should be a dictionary"
self.persistence_integrity_check()
if linked and linked.link_public_key != ANY_COUNTERPARTY_PK:
block_type = linked.type
block = self.get_block_class(block_type).create(
block_type,
transaction,
self.persistence,
self.my_peer.public_key.key_to_bin(),
link=linked,
additional_info=additional_info,
link_pk=public_key)
block.sign(self.my_peer.key)
validation = block.validate(self.persistence)
self.logger.info("Signed block to %s (%s) validation result %s",
hexlify(block.link_public_key)[-8:], block,
validation)
if validation[0] != ValidationResult.partial_next and validation[
0] != ValidationResult.valid:
self.logger.error("Signed block did not validate?! Result %s",
repr(validation))
return fail(RuntimeError("Signed block did not validate."))
if not self.persistence.contains(block):
self.persistence.add_block(block)
self.notify_listeners(block)
# This is a source block with no counterparty
if not peer and public_key == ANY_COUNTERPARTY_PK:
if block.type not in self.settings.block_types_bc_disabled:
self.send_block(block)
return succeed((block, None))
# If there is a counterparty to sign, we send it
self.send_block(block, address=peer.address)
# We broadcast the block in the network if we initiated a transaction
if block.type not in self.settings.block_types_bc_disabled and not linked:
self.send_block(block)
if peer == self.my_peer:
# We created a self-signed block
if block.type not in self.settings.block_types_bc_disabled:
self.send_block(block)
return succeed(
(block,
None)) if public_key == ANY_COUNTERPARTY_PK else succeed(
(block, linked))
elif not linked:
# We keep track of this outstanding sign request.
sign_deferred = Deferred()
self.request_cache.add(
HalfBlockSignCache(self, block, sign_deferred, peer.address))
return sign_deferred
else:
# We return a deferred that fires immediately with both half blocks.
if block.type not in self.settings.block_types_bc_disabled:
self.send_block_pair(linked, block)
return succeed((linked, block))
def xmlrpc_defer_fail(self):
"""
Show how xmlrpc methods can return failed Deferred.
"""
return defer.fail(12)
def xmlrpc_deferFail(self):
return defer.fail(TestValueError())
def on_start_fail(result):
self._component_state = 'Stopped'
self._component_starting_deferred = None
log.error(result)
return fail(result)
def on_balance(balance):
if self.check_negative_balance and balance['available'] < quantity:
return fail(InsufficientFunds())
return self.create_transfer_block(peer, quantity)
def queryProtocol(self, *args, **kwargs):
return defer.fail(socket.gaierror("Couldn't connect"))
def test_failure(self):
# extract_result on a Deferred that has an error raises the failing
# exception.
deferred = defer.fail(RuntimeError())
self.assertRaises(RuntimeError, extract_result, deferred)