def find(self, wordlist):
'''look up all the words in the wordlist.
If none are found return an empty dictionary
* more rules here
'''
if not wordlist:
return {}
database = self._get_database()
enquire = xapian.Enquire(database)
stemmer = xapian.Stem("english")
terms = []
for term in [
word.upper() for word in wordlist
if self.minlength <= len(word) <= self.maxlength
]:
if not self.is_stopword(term):
terms.append(stemmer(s2b(term.lower())))
query = xapian.Query(xapian.Query.OP_AND, terms)
enquire.set_query(query)
matches = enquire.get_mset(0, database.get_doccount())
return [tuple(b2s(m.document.get_data()).split(':')) for m in matches]
def ssha(password, salt):
''' Make ssha digest from password and salt.
Based on code of Roberto Aguilar <*****@*****.**>
https://gist.github.com/rca/7217540
'''
shaval = sha1(password) # nosec
shaval.update(salt)
ssha_digest = b2s(b64encode(shaval.digest() + salt).strip())
return ssha_digest
def test_compression_none_etag(self):
# use basic auth for rest endpoint
f = requests.get(self.url_base() + '/rest/data/user/1/username',
auth=('admin', 'sekrit'),
headers={
'content-type': "",
'Accept-Encoding': "",
'Accept': '*/*'
})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = {
'Content-Type': 'application/json',
'Access-Control-Allow-Credentials': 'true',
'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
}
content_str = '''{ "data": {
"id": "1",
"link": "http://*****:*****@etag'])
# type is "class 'str'" under py3, "type 'str'" py2
# just skip comparing it.
del (json_dict['data']['type'])
self.assertDictEqual(json_dict, content)
# verify that ETag header has no - delimiter
print(f.headers['ETag'])
with self.assertRaises(ValueError):
f.headers['ETag'].index('-')
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
def test_rest_preflight_collection(self):
# no auth for rest csrf preflight
f = requests.options(self.url_base() + '/rest/data/user',
headers={
'content-type': "",
'x-requested-with': "rest",
'Access-Control-Request-Headers':
"x-requested-with",
'Access-Control-Request-Method': "PUT",
'Origin': "https://client.com"
})
print(f.status_code)
print(f.headers)
print(f.content)
self.assertEqual(f.status_code, 204)
expected = {
'Access-Control-Allow-Origin': 'https://client.com',
'Access-Control-Allow-Headers':
'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override',
'Allow': 'OPTIONS, GET, POST',
'Access-Control-Allow-Methods': 'OPTIONS, GET, POST',
'Access-Control-Allow-Credentials': 'true',
}
# use dict comprehension to filter headers to the ones we want to check
self.assertEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
# use invalid Origin
f = requests.options(self.url_base() + '/rest/data/user',
headers={
'content-type': "application/json",
'x-requested-with': "rest",
'Access-Control-Request-Headers':
"x-requested-with",
'Access-Control-Request-Method': "PUT",
'Origin': "ZZZ"
})
self.assertEqual(f.status_code, 400)
expected = '{ "error": { "status": 400, "msg": "Client is not ' \
'allowed to use Rest Interface." } }'
self.assertEqual(b2s(f.content), expected)
def nice_sender_header(name, address, charset):
# construct an address header so it's as human-readable as possible
# even in the presence of a non-ASCII name part
if not name:
return address
try:
encname = b2s(name.encode('ASCII'))
except UnicodeEncodeError:
# use Header to encode correctly.
encname = Header(name, charset=charset).encode()
# the important bits of formataddr()
if specialsre.search(encname):
encname = '"%s"' % escapesre.sub(r'\\\g<0>', encname)
# now format the header as a string - don't return a Header as anonymous
# headers play poorly with Messages (eg. won't get wrapped properly)
return '%s <%s>' % (encname, address)
def h64encode(data):
"""encode using variant of base64"""
return b2s(b64encode(data, b"./").strip(b"=\n"))
def pack_timestamp():
return b2s(base64.b64encode(struct.pack("i", int(time.time()))).strip())
def test_compression_gzip(self, method='gzip'):
if method == 'gzip':
decompressor = None
elif method == 'br':
decompressor = brotli.decompress
elif method == 'zstd':
decompressor = zstd.decompress
# use basic auth for rest endpoint
f = requests.get(self.url_base() + '/rest/data/user/1/username',
auth=('admin', 'sekrit'),
headers={
'content-type': "",
'Accept-Encoding': '%s, foo' % method,
'Accept': '*/*'
})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = {
'Content-Type': 'application/json',
'Access-Control-Allow-Credentials': 'true',
'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
'Content-Encoding': method,
'Vary': 'Origin, Accept-Encoding',
}
content_str = '''{ "data": {
"id": "1",
"link": "http://*****:*****@etag'])
# type is "class 'str'" under py3, "type 'str'" py2
# just skip comparing it.
del (json_dict['data']['type'])
self.assertDictEqual(json_dict, content)
# verify that ETag header ends with -<method>
try:
self.assertRegex(f.headers['ETag'],
r'^"[0-9a-f]{32}-%s"$' % method)
except AttributeError:
# python2 no assertRegex so try substring match
self.assertEqual(33, f.headers['ETag'].rindex('-' + method))
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
# use basic auth for rest endpoint, error case, bad attribute
f = requests.get(self.url_base() + '/rest/data/user/1/foo',
auth=('admin', 'sekrit'),
headers={
'content-type': "",
'Accept-Encoding': '%s, foo' % method,
'Accept': '*/*',
'Origin': 'ZZZZ'
})
print(f.status_code)
print(f.headers)
# NOTE: not compressed payload too small
self.assertEqual(f.status_code, 400)
expected = {
'Content-Type': 'application/json',
'Access-Control-Allow-Credentials': 'true',
'Access-Control-Allow-Origin': 'ZZZZ',
'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH',
'Vary': 'Origin'
}
content = {"error": {"status": 400, "msg": "Invalid attribute foo"}}
json_dict = json.loads(b2s(f.content))
self.assertDictEqual(json_dict, content)
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
# test file x-fer
f = requests.get(self.url_base() + '/@@file/user_utils.js',
headers={
'Accept-Encoding': '%s, foo' % method,
'Accept': '*/*'
})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = {
'Content-Type': 'application/javascript',
'Content-Encoding': method,
'Vary': 'Accept-Encoding',
}
# compare to byte string as f.content may be compressed.
# so running b2s on it will throw a UnicodeError
if f.content[0:25] == b'// User Editing Utilities':
# no need to decompress, urlib3.response did it for gzip and br
data = f.content
else:
# I need to decode
data = decompressor(f.content)
# check first few bytes.
self.assertEqual(b2s(data)[0:25], '// User Editing Utilities')
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
# test file x-fer
f = requests.get(self.url_base() + '/user1',
headers={
'Accept-Encoding': '%s, foo' % method,
'Accept': '*/*'
})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = {
'Content-Type': 'text/html; charset=utf-8',
'Content-Encoding': method,
'Vary': 'Accept-Encoding',
}
if f.content[0:25] == b'<!-- dollarId: user.item,':
# no need to decompress, urlib3.response did it for gzip and br
data = f.content
else:
# I need to decode
data = decompressor(f.content)
# check first few bytes.
self.assertEqual(b2s(data[0:25]), '<!-- dollarId: user.item,')
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
def test_compression_gzipfile(self):
'''Get the compressed dummy file'''
# create a user_utils.js.gz file to test pre-compressed
# file serving code. Has custom contents to verify
# that I get the compressed one.
gzfile = "%s/html/user_utils.js.gzip" % self.dirname
test_text = b"Custom text for user_utils.js\n"
with gzip.open(gzfile, 'wb') as f:
bytes_written = f.write(test_text)
self.assertEqual(bytes_written, 30)
# test file x-fer
f = requests.get(self.url_base() + '/@@file/user_utils.js',
headers={
'Accept-Encoding': 'gzip, foo',
'Accept': '*/*'
})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = {
'Content-Type': 'application/javascript',
'Content-Encoding': 'gzip',
'Vary': 'Accept-Encoding',
'Content-Length': '69',
}
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
# check content - verify it's the .gz file not the real file.
self.assertEqual(f.content, test_text)
'''# verify that a different encoding request returns on the fly
# test file x-fer using br, so we get runtime compression
f = requests.get(self.url_base() + '/@@file/user_utils.js',
headers = { 'Accept-Encoding': 'br, foo',
'Accept': '*/*'})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = { 'Content-Type': 'application/javascript',
'Content-Encoding': 'br',
'Vary': 'Accept-Encoding',
'Content-Length': '960',
}
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual({ key: value for (key, value) in
f.headers.items() if key in expected },
expected)
try:
from urllib3.response import BrotliDecoder
# requests has decoded br to text for me
data = f.content
except ImportError:
# I need to decode
data = brotli.decompress(f.content)
self.assertEqual(b2s(data)[0:25], '// User Editing Utilities')
'''
# re-request file, but now make .gzip out of date. So we get the
# real file compressed on the fly, not our test file.
os.utime(gzfile, (0, 0)) # use 1970/01/01 or os base time
f = requests.get(self.url_base() + '/@@file/user_utils.js',
headers={
'Accept-Encoding': 'gzip, foo',
'Accept': '*/*'
})
print(f.status_code)
print(f.headers)
self.assertEqual(f.status_code, 200)
expected = {
'Content-Type': 'application/javascript',
'Content-Encoding': 'gzip',
'Vary': 'Accept-Encoding',
}
# use dict comprehension to remove fields like date,
# content-length etc. from f.headers.
self.assertDictEqual(
{
key: value
for (key, value) in f.headers.items() if key in expected
}, expected)
# check content - verify it's the real file, not crafted .gz.
self.assertEqual(b2s(f.content)[0:25], '// User Editing Utilities')
# cleanup
os.remove(gzfile)
def send_message(self,
issueid,
msgid,
note,
sendto,
from_address=None,
bcc_sendto=[],
subject=None,
crypt=False,
add_headers={},
authid=None):
'''Actually send the nominated message from this issue to the sendto
recipients, with the note appended. It's possible to add
headers to the message with the add_headers variable.
'''
users = self.db.user
messages = self.db.msg
files = self.db.file
if msgid is None:
inreplyto = None
messageid = None
else:
inreplyto = messages.get(msgid, 'inreplyto')
messageid = messages.get(msgid, 'messageid')
# make up a messageid if there isn't one (web edit)
if not messageid:
# this is an old message that didn't get a messageid, so
# create one
messageid = "<%s.%s.%s%s@%s>" % (
time.time(), b2s(base64.b32encode(random_.token_bytes(10))),
self.classname, issueid, self.db.config['MAIL_DOMAIN'])
if msgid is not None:
messages.set(msgid, messageid=messageid)
# compose title
cn = self.classname
title = self.get(issueid, 'title') or '%s message copy' % cn
# figure author information
if authid:
pass
elif msgid:
authid = messages.get(msgid, 'author')
else:
authid = self.db.getuid()
authname = users.get(authid, 'realname')
if not authname:
authname = users.get(authid, 'username', '')
authaddr = users.get(authid, 'address', '')
if authaddr and self.db.config.MAIL_ADD_AUTHOREMAIL:
authaddr = " <%s>" % formataddr(('', authaddr))
elif authaddr:
authaddr = ""
# make the message body
m = ['']
# put in roundup's signature
if self.db.config.EMAIL_SIGNATURE_POSITION == 'top':
m.append(self.email_signature(issueid, msgid))
# add author information
if authid and self.db.config.MAIL_ADD_AUTHORINFO:
if msgid and len(self.get(issueid, 'messages')) == 1:
m.append(
_("New submission from %(authname)s%(authaddr)s:") %
locals())
elif msgid:
m.append(
_("%(authname)s%(authaddr)s added the comment:") %
locals())
else:
m.append(_("Change by %(authname)s%(authaddr)s:") % locals())
m.append('')
# add the content
if msgid is not None:
m.append(messages.get(msgid, 'content', ''))
# get the files for this message
message_files = []
if msgid:
for fileid in messages.get(msgid, 'files'):
# check the attachment size
filesize = self.db.filesize('file', fileid, None)
if filesize <= self.db.config.NOSY_MAX_ATTACHMENT_SIZE:
message_files.append(fileid)
else:
base = self.db.config.TRACKER_WEB
link = "".join((base, files.classname, fileid))
filename = files.get(fileid, 'name')
m.append(
_("File '%(filename)s' not attached - "
"you can download it from %(link)s.") % locals())
# add the change note
if note:
m.append(note)
# put in roundup's signature
if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom':
m.append(self.email_signature(issueid, msgid))
# figure the encoding
charset = getattr(self.db.config, 'EMAIL_CHARSET', 'utf-8')
# construct the content and convert to unicode object
body = s2u('\n'.join(m))
# make sure the To line is always the same (for testing mostly)
sendto.sort()
# make sure we have a from address
if from_address is None:
from_address = self.db.config.TRACKER_EMAIL
# additional bit for after the From: "name"
from_tag = getattr(self.db.config, 'EMAIL_FROM_TAG', '')
if from_tag:
from_tag = ' ' + from_tag
if subject is None:
subject = '[%s%s] %s' % (cn, issueid, title)
author = (authname + from_tag, from_address)
# send an individual message per recipient?
if self.db.config.NOSY_EMAIL_SENDING != 'single':
sendto = [[address] for address in sendto]
else:
sendto = [sendto]
# tracker sender info
tracker_name = s2u(self.db.config.TRACKER_NAME)
tracker_name = nice_sender_header(tracker_name, from_address, charset)
# now send one or more messages
# TODO: I believe we have to create a new message each time as we
# can't fiddle the recipients in the message ... worth testing
# and/or fixing some day
first = True
for sendto in sendto:
# create the message
mailer = Mailer(self.db.config)
message = mailer.get_standard_message(multipart=message_files)
# set reply-to as requested by config option
# TRACKER_REPLYTO_ADDRESS
replyto_config = self.db.config.TRACKER_REPLYTO_ADDRESS
if replyto_config:
if replyto_config == "AUTHOR":
# note that authaddr at this point is already
# surrounded by < >, so get the original address
# from the db as nice_send_header adds < >
replyto_addr = nice_sender_header(
authname, users.get(authid, 'address', ''), charset)
else:
replyto_addr = replyto_config
else:
replyto_addr = tracker_name
message['Reply-To'] = replyto_addr
# message ids
if messageid:
message['Message-Id'] = messageid
if inreplyto:
message['In-Reply-To'] = inreplyto
# Generate a header for each link or multilink to
# a class that has a name attribute
for propname, prop in self.getprops().items():
if not isinstance(prop, (hyperdb.Link, hyperdb.Multilink)):
continue
cl = self.db.getclass(prop.classname)
label = None
if 'name' in cl.getprops():
label = 'name'
if prop.msg_header_property in cl.getprops():
label = prop.msg_header_property
if prop.msg_header_property == "":
# if msg_header_property is set to empty string
# suppress the header entirely. You can't use
# 'msg_header_property == None'. None is the
# default value.
label = None
if not label:
continue
if isinstance(prop, hyperdb.Link):
value = self.get(issueid, propname)
if value is None:
continue
values = [value]
else:
values = self.get(issueid, propname)
if not values:
continue
values = [cl.get(v, label) for v in values]
values = ', '.join(values)
header = "X-Roundup-%s-%s" % (self.classname, propname)
try:
values.encode('ascii')
message[header] = values
except UnicodeError:
message[header] = Header(values, charset)
# Add header for main id number to make filtering
# email easier than extracting from subject line.
header = "X-Roundup-%s-Id" % (self.classname)
values = issueid
try:
values.encode('ascii')
message[header] = values
except UnicodeError:
message[header] = Header(values, charset)
# Generate additional headers
for k in add_headers:
v = add_headers[k]
try:
v.encode('ascii')
message[k] = v
except UnicodeError:
message[k] = Header(v, charset)
if not inreplyto:
# Default the reply to the first message
msgs = self.get(issueid, 'messages')
# Assume messages are sorted by increasing message number here
# If the issue is just being created, and the submitter didn't
# provide a message, then msgs will be empty.
if msgs and msgs[0] != msgid:
inreplyto = messages.get(msgs[0], 'messageid')
if inreplyto:
message['In-Reply-To'] = inreplyto
# attach files
if message_files:
# first up the text as a part
part = mailer.get_standard_message()
part.set_payload(body, part.get_charset())
message.attach(part)
for fileid in message_files:
name = files.get(fileid, 'name')
mime_type = (files.get(fileid, 'type')
or mimetypes.guess_type(name)[0]
or 'application/octet-stream')
if mime_type == 'text/plain':
content = files.get(fileid, 'content')
part = MIMEText('')
del part['Content-Transfer-Encoding']
try:
enc = content.encode('ascii')
part = mailer.get_text_message('us-ascii')
part.set_payload(enc)
except UnicodeError:
# the content cannot be 7bit-encoded.
# use quoted printable
# XXX stuffed if we know the charset though :(
part = mailer.get_text_message('utf-8')
part.set_payload(content, part.get_charset())
elif mime_type == 'message/rfc822':
content = files.get(fileid, 'content')
main, sub = mime_type.split('/')
p = FeedParser()
p.feed(content)
part = MIMEBase(main, sub)
part.set_payload([p.close()])
else:
# some other type, so encode it
content = files.get(fileid, 'binary_content')
main, sub = mime_type.split('/')
part = MIMEBase(main, sub)
part.set_payload(content)
encoders.encode_base64(part)
cd = 'Content-Disposition'
part[cd] = 'attachment;\n filename="%s"' % name
message.attach(part)
else:
message.set_payload(body, message.get_charset())
if crypt:
send_msg = self.encrypt_to(message, sendto)
else:
send_msg = message
mailer.set_message_attributes(send_msg, sendto, subject, author)
if crypt:
send_msg['Message-Id'] = message['Message-Id']
send_msg['Reply-To'] = message['Reply-To']
if message.get('In-Reply-To'):
send_msg['In-Reply-To'] = message['In-Reply-To']
if sendto:
mailer.smtp_send(sendto, send_msg.as_string())
if first:
if crypt:
# send individual bcc mails, otherwise receivers can
# deduce bcc recipients from keys in message
for bcc in bcc_sendto:
send_msg = self.encrypt_to(message, [bcc])
send_msg['Message-Id'] = message['Message-Id']
send_msg['Reply-To'] = message['Reply-To']
if message.get('In-Reply-To'):
send_msg['In-Reply-To'] = message['In-Reply-To']
mailer.smtp_send([bcc], send_msg.as_string())
elif bcc_sendto:
mailer.smtp_send(bcc_sendto, send_msg.as_string())
first = False
