def roster_update_users(self, e, sr):
'''Update users' full names and invert hash
For all *users* we have information about:
- collect the shared roster groups they belong to
- set their full names if not yet defined
Return inverted hash'''
groups = {}
commands = []
for user, desc in sr.items():
if 'groups' in desc:
for g in desc['groups']:
if g in groups:
groups[g].append(user)
else:
groups[g] = [user]
if 'name' in desc:
lhs, rhs = self.jidsplit(user)
fnc = utf8('FNC:' + user) # No unicode keys
if fnc in self.ctx.shared_roster_db:
cached_name = unutf8(self.ctx.shared_roster_db[fnc])
else:
cached_name = None
self.ctx.shared_roster_db[fnc] = utf8(desc['name'])
cmd = e.maybe_set_fn(lhs,
rhs,
desc['name'],
cached_name=cached_name)
if cmd is not None:
commands.append(cmd)
return groups, commands
class roster(roster_thread):
def jidsplit(self, jid):
'''Split jid into lhs@rhs'''
(node, at, dom) = jid.partition('@')
if at == '':
return (node, self.domain)
else:
return (node, dom)
def roster_cloud(self):
'''Query roster JSON from cloud'''
success, code, message, text = self.verbose_cloud_request({
'operation': 'sharedroster',
'username': self.username,
'domain': self.authDomain
})
if success:
if code is not None and code != requests.codes.ok:
return code, None
else:
sr = None
try:
sr = message['data']['sharedRoster']
return sr, text
except Exception as e:
logging.warn('Weird response: ' + str(e))
return message, text
else:
return False, None
def try_roster(self, async=True):
'''Maybe update roster'''
if (self.ctx.ejabberdctl_path is not None):
try:
response, text = self.roster_cloud()
if response is not None and response != False:
texthash = hashlib.sha256(utf8(text)).hexdigest()
userhash = utf8('RH:' + self.username + ':' + self.domain)
# Response changed or first response for that user?
if not userhash in self.ctx.shared_roster_db or self.ctx.shared_roster_db[userhash] != texthash:
self.ctx.shared_roster_db[userhash] = texthash
t = threading.Thread(target=self.roster_background_thread,
args=[response])
t.start()
if not async:
t.join() # For automated testing only
else:
# Try to do most before the user is actually logged in.
# Thanks to improved caching, this should rarely be noticeable
# and reduce the 'full names only visible on second login'
# problem experienced especially in Gajim (maybe a race condition?)
t.join(1.0)
return True
except Exception as err:
(etype, value, tb) = sys.exc_info()
traceback.print_exception(etype, value, tb)
logging.warn('roster_groups thread: %s:\n%s'
% (str(err), ''.join(traceback.format_tb(tb))))
return False
return True
def roster_update_groups(self, e, groups):
'''Update shared roster groups with ejabberdctl
For all the *groups* we have information about:
- create the group (idempotent)
- delete the users that we do not know about anymore
- add the users we know about (idempotent)'''
cleanname = {}
for g in groups:
cleanname[g] = sanitize(g)
key = utf8('RGC:%s:%s' % (cleanname[g], self.domain))
if key in self.ctx.shared_roster_db:
previous_users = self.ctx.shared_roster_db[key].split('\t')
else:
e.execute([
'srg_create', cleanname[g], self.domain, cleanname[g],
cleanname[g], cleanname[g]
])
# Fill cache (again)
previous_users = e.members(cleanname[g], self.domain)
new_users = {}
for u in groups[g]:
(lhs, rhs) = self.jidsplit(u)
fulljid = '%s@%s' % (lhs, rhs)
new_users[fulljid] = True
if not fulljid in previous_users:
e.execute(
['srg_user_add', lhs, rhs, cleanname[g], self.domain])
for p in previous_users:
(lhs, rhs) = self.jidsplit(p)
if p not in new_users:
e.execute(
['srg_user_del', lhs, rhs, cleanname[g], self.domain])
self.ctx.shared_roster_db[key] = '\t'.join(sorted(
new_users.keys()))
# For all the groups the login user was previously a member of:
# - delete her from the shared roster group if no longer a member
key = utf8('LIG:%s@%s' % (self.username, self.domain))
if key in self.ctx.shared_roster_db and self.ctx.shared_roster_db[
key] != '':
# Was previously there as well, need to be removed from one?
previous = self.ctx.shared_roster_db[key].split('\t')
for p in previous:
if p not in list(cleanname.values()):
e.execute([
'srg_user_del', self.username, self.domain, p,
self.domain
])
# Only update when necessary
if not cleanname:
del self.ctx.shared_roster_db[key]
else:
new = '\t'.join(sorted(cleanname.values()))
if previous != new:
self.ctx.shared_roster_db[key] = new
else: # New, always set
if cleanname:
self.ctx.shared_roster_db[key] = '\t'.join(
sorted(cleanname.values()))
def checkpw(self, pwhash):
'''Compare self.password with pwhash.
Try to be resistant to timing attacks and use `checkpw` if available.'''
pw = utf8(self.password)
pwhash = utf8(pwhash)
if 'checkpw' in dir(bcrypt):
return bcrypt.checkpw(pw, pwhash)
else:
ret = bcrypt.hashpw(pw, pwhash)
return ret == pwhash
def per_domain(self, dom):
bdom = utf8(dom)
if bdom in self.domain_db:
try:
# Already 4-value database format? Great!
secret, url, authDomain, extra = self.domain_db[bdom].split('\t', 3)
except ValueError:
# No, fall back to 3-value format (and update DB)
secret, url, extra = self.domain_db[bdom].split('\t', 2)
authDomain = dom
self.domain_db[dom] = '\t'.join((secret, url, authDomain, extra))
return utf8(secret), url, authDomain
else:
return utf8(self.default_secret), self.default_url, dom
def auth_update_cache(self):
if self.ctx.db.cache_storage == 'none':
return False
jid = self.username + '@' + self.domain
now = self.now
try:
if self.ctx.db.cache_storage == 'memory':
rounds = self.ctx.bcrypt_rounds[1]
else:
rounds = self.ctx.bcrypt_rounds[0]
salt = bcrypt.gensalt(rounds=rounds)
except TypeError:
# Old versions of bcrypt() apparently do not support the rounds option
salt = bcrypt.gensalt()
pwhash = unutf8(bcrypt.hashpw(utf8(self.password), salt))
# Upsert in SQLite is too new to rely on:
# https://www.sqlite.org/draft/lang_UPSERT.html
#
# INSERT OR REPLACE cannot be used, as it will inherit
# the DEFAULT values instead of the existing values.
self.ctx.db.cache.begin()
self.ctx.db.cache.execute(
'''INSERT OR IGNORE INTO authcache (jid, firstauth)
VALUES (?, ?)''',
(jid, now))
self.ctx.db.cache.execute(
'''UPDATE authcache
SET pwhash = ?, remoteauth = ?, anyauth = ?
WHERE jid = ?''', (pwhash, now, now, jid))
self.ctx.db.cache.commit()
def auth_token(self):
try:
token = b64decode(
self.password.translate(usersafe_encoding) + '=======')
except:
logging.debug('Could not decode token (maybe not a token?)')
return False
jid = self.username + '@' + self.domain
if len(token) != 23:
logging.debug('Token is too short: %d != 23 (maybe not a token?)' %
len(token))
return False
(version, mac, header) = unpack('> B 16s 6s', token)
if version != 0:
logging.debug('Wrong token version (maybe not a token?)')
return False
(secretID, expiry) = unpack('> H I', header)
if expiry < self.now:
logging.debug('Token has expired')
return False
challenge = pack('> B 6s %ds' % len(jid), version, header, utf8(jid))
response = hmac.new(self.secret, challenge, hashlib.sha256).digest()
return hmac.compare_digest(mac, response[:16])
def auth_token(self):
try:
token = b64decode(self.password.translate(usersafe_encoding) + '=======')
except:
logging.debug('Not a token (not base64)')
return False
jid = self.username + '@' + self.domain
if len(token) != 23:
logging.debug('Not a token (len: %d != 23)' % len(token))
return False
(version, mac, header) = unpack('> B 16s 6s', token)
if version != 0:
logging.debug('Not a token (version: %d != 0)' % version)
return False;
(secretID, expiry) = unpack('> H I', header)
expiry = datetime.utcfromtimestamp(expiry)
if expiry < self.now:
logging.debug('Token has expired')
return False
challenge = pack('> B 6s %ds' % len(jid), version, header, utf8(jid))
response = hmac.new(self.secret, challenge, hashlib.sha256).digest()
if hmac.compare_digest(mac, response[:16]):
return True
else:
logging.warning('Token for %s has invalid signature (possible attack attempt!)' % jid)
return False
def perform(args):
domain_db = bsddb3.hashopen(args.domain_db, 'c', 0o600)
if args.get:
print(unutf8(domain_db[utf8(args.get)]))
elif args.put:
domain_db[utf8(args.put[0])] = args.put[1]
elif args.delete:
del domain_db[utf8(args.delete)]
elif args.unload:
for k in list(domain_db.keys()):
print('%s\t%s' % (unutf8(k), unutf8(domain_db[k])))
# Should work according to documentation, but doesn't
# for k, v in DOMAIN_DB.iteritems():
# print k, '\t', v
elif args.load:
for line in sys.stdin:
k, v = line.rstrip('\r\n').split('\t', 1)
domain_db[utf8(k)] = v
domain_db.close()
def test_05_unload(self):
expected = [b'example.net', b'example.ch']
self.stub_stdout(ioclass=io.StringIO)
ns = self.mkns(unload=True)
perform(ns)
v = sys.stdout.getvalue()
for line in v.split('\n'):
if line != '':
(k, delim, v) = line.partition('\t')
expected.remove(utf8(k))
assertEqual(expected, [])
def verbose_cloud_request(self, data):
'''Perform a signed cloud request on data with detailed result.
Return tuple:
- (True, None, json, body): Remote side answered with HTTP 200 and JSON body
- (False, 200, None, None): Remote side answered with HTTP 200, but no JSON
- (False, int, json, body): Remote side answered != 200, with JSON body
- (False, int, None, None): Remote side answered != 200, without JSON
- (False, None, err, None): Connection problem, described in err
'''
# logging.debug("Sending %s to %s" % (data, url))
payload = utf8(urllib.parse.urlencode(data))
signature = hmac.new(self.secret, msg=payload,
digestmod=hashlib.sha1).hexdigest()
headers = {
'X-JSXC-SIGNATURE': 'sha1=' + signature,
'content-type': 'application/x-www-form-urlencoded'
}
try:
r = self.ctx.session.post(self.url,
data=payload,
headers=headers,
allow_redirects=False,
timeout=self.ctx.timeout)
except requests.exceptions.HTTPError as err:
logging.warn(err)
return False, None, err, None
except requests.exceptions.RequestException as err:
try:
logging.warn(
'An error occured during the request to %s for domain %s: %s'
% (self.url, data['domain'], err))
except TypeError as err:
logging.warn(
'An unknown error occured during the request to %s, probably an SSL error. Try updating your "requests" and "urllib" libraries.'
% url)
return False, None, err, None
if r.status_code != requests.codes.ok:
try:
return False, r.status_code, r.json(), r.text
except ValueError: # Not a valid JSON response
return False, r.status_code, None, None
try:
# Return True only for HTTP 200 with JSON body, False for everything else
return True, None, r.json(), r.text
except ValueError: # Not a valid JSON response
return False, r.status_code, None, None
def try_roster(self, async_=True):
'''Maybe update roster'''
if (self.ctx.ejabberdctl_path is not None):
try:
response, text = self.roster_cloud()
if response is not None and response != False:
jid = '@'.join((self.username, self.domain))
texthash = hashlib.sha256(utf8(text)).hexdigest()
# Response changed or first response for that user?
cache_valid = False
for row in self.ctx.db.conn.execute(
'SELECT responsehash FROM rosterinfo where jid=?',
(jid, )):
if row['responsehash'] == texthash:
cache_valid = True
if not cache_valid:
self.ctx.db.conn.begin()
self.ctx.db.conn.execute(
'''INSERT OR IGNORE
INTO rosterinfo (jid)
VALUES (?)''', (jid, ))
self.ctx.db.conn.execute(
'''UPDATE rosterinfo
SET responsehash = ?, last_update = ?
WHERE jid = ?''',
(texthash, datetime.utcnow(), jid))
self.ctx.db.conn.commit()
t = threading.Thread(
target=self.roster_background_thread,
args=(response, ))
t.start()
if not async_:
t.join() # For automated testing only
else:
# Try to do most before the user is actually logged in.
# Thanks to improved caching, this should rarely be noticeable
# and reduce the 'full names only visible on second login'
# problem experienced especially in Gajim (maybe a race condition?)
t.join(1.0)
return True
except Exception as err:
(etype, value, tb) = sys.exc_info()
traceback.print_exception(etype, value, tb)
logging.warn('roster_groups thread: %s:\n%s' %
(str(err), ''.join(traceback.format_tb(tb))))
return False
return True
def auth_update_cache(self):
if '' in self.ctx.cache_db: # Cache disabled?
return
key = self.username + ':' + self.domain
now = self.now # For tests
snow = str(now)
try:
salt = bcrypt.gensalt(rounds=self.ctx.bcrypt_rounds)
except TypeError:
# Old versions of bcrypt() apparently do not support the rounds option
salt = bcrypt.gensalt()
pwhash = unutf8(bcrypt.hashpw(utf8(self.password), salt))
if key in self.ctx.cache_db:
(ignored, ts1, tsv, tsa,
rest) = self.ctx.cache_db[key].split("\t", 4)
self.ctx.cache_db[key] = "\t".join((pwhash, ts1, snow, snow, rest))
else:
self.ctx.cache_db[key] = "\t".join((pwhash, snow, snow, snow, ''))
self.try_db_sync()
class roster(roster_thread):
def jidsplit(self, jid):
'''Split jid into lhs@rhs'''
(node, at, dom) = jid.partition('@')
if at == '':
return (node, self.domain)
else:
return (node, dom)
def roster_cloud(self):
'''Query roster JSON from cloud'''
success, code, message, text = self.verbose_cloud_request({
'operation':
'sharedroster',
'username':
self.username,
'domain':
self.authDomain
})
if success:
if code is not None and code != requests.codes.ok:
return code, None
else:
sr = None
try:
sr = message['data']['sharedRoster']
return sr, text
except Exception as e:
logging.warn('Weird response: ' + str(e))
return message, text
else:
return False, None
def try_roster(self, async=True):
'''Maybe update roster'''
if (self.ctx.ejabberdctl_path is not None):
try:
response, text = self.roster_cloud()
if response is not None and response != False:
jid = '@'.join((self.username, self.domain))
texthash = hashlib.sha256(utf8(text)).hexdigest()
# Response changed or first response for that user?
cache_valid = False
for row in self.ctx.db.conn.execute(
'SELECT responsehash FROM rosterinfo where jid=?',
(jid, )):
if row['responsehash'] == texthash:
cache_valid = True
if not cache_valid:
self.ctx.db.conn.begin()
self.ctx.db.conn.execute(
'''INSERT OR IGNORE
INTO rosterinfo (jid)
VALUES (?)''', (jid, ))
self.ctx.db.conn.execute(
'''UPDATE rosterinfo
SET responsehash = ?, last_update = ?
WHERE jid = ?''',
(texthash, datetime.utcnow(), jid))
self.ctx.db.conn.commit()
t = threading.Thread(
target=self.roster_background_thread,
args=(response, ))
t.start()
if not async:
t.join() # For automated testing only
else:
# Try to do most before the user is actually logged in.
# Thanks to improved caching, this should rarely be noticeable
# and reduce the 'full names only visible on second login'
# problem experienced especially in Gajim (maybe a race condition?)
t.join(1.0)
return True
except Exception as err:
(etype, value, tb) = sys.exc_info()
traceback.print_exception(etype, value, tb)
logging.warn('roster_groups thread: %s:\n%s' %
(str(err), ''.join(traceback.format_tb(tb))))
return False
return True
def test_utf8_ascii():
assertEqual(b'hallo', utf8(u'hallo'))
def test_utf8_valid():
assertEqual(b'Hall\xc3\xb6chen', utf8(u'Hallöchen'))
def per_domain(self, dom):
for row in self.db.conn.execute('SELECT authsecret, authurl, authdomain FROM domains WHERE xmppdomain = ?', (dom,)):
return utf8(row[0]), row[1], row[2]
return utf8(self.default_secret), self.default_url, dom