def find(self, wordlist): '''look up all the words in the wordlist. If none are found return an empty dictionary * more rules here ''' if not wordlist: return {} database = self._get_database() enquire = xapian.Enquire(database) stemmer = xapian.Stem("english") terms = [] for term in [ word.upper() for word in wordlist if self.minlength <= len(word) <= self.maxlength ]: if not self.is_stopword(term): terms.append(stemmer(s2b(term.lower()))) query = xapian.Query(xapian.Query.OP_AND, terms) enquire.set_query(query) matches = enquire.get_mset(0, database.get_doccount()) return [tuple(b2s(m.document.get_data()).split(':')) for m in matches]
def ssha(password, salt): ''' Make ssha digest from password and salt. Based on code of Roberto Aguilar <*****@*****.**> https://gist.github.com/rca/7217540 ''' shaval = sha1(password) # nosec shaval.update(salt) ssha_digest = b2s(b64encode(shaval.digest() + salt).strip()) return ssha_digest
def test_compression_none_etag(self): # use basic auth for rest endpoint f = requests.get(self.url_base() + '/rest/data/user/1/username', auth=('admin', 'sekrit'), headers={ 'content-type': "", 'Accept-Encoding': "", 'Accept': '*/*' }) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'application/json', 'Access-Control-Allow-Credentials': 'true', 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', } content_str = '''{ "data": { "id": "1", "link": "http://*****:*****@etag']) # type is "class 'str'" under py3, "type 'str'" py2 # just skip comparing it. del (json_dict['data']['type']) self.assertDictEqual(json_dict, content) # verify that ETag header has no - delimiter print(f.headers['ETag']) with self.assertRaises(ValueError): f.headers['ETag'].index('-') # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected)
def test_rest_preflight_collection(self): # no auth for rest csrf preflight f = requests.options(self.url_base() + '/rest/data/user', headers={ 'content-type': "", 'x-requested-with': "rest", 'Access-Control-Request-Headers': "x-requested-with", 'Access-Control-Request-Method': "PUT", 'Origin': "https://client.com" }) print(f.status_code) print(f.headers) print(f.content) self.assertEqual(f.status_code, 204) expected = { 'Access-Control-Allow-Origin': 'https://client.com', 'Access-Control-Allow-Headers': 'Content-Type, Authorization, X-Requested-With, X-HTTP-Method-Override', 'Allow': 'OPTIONS, GET, POST', 'Access-Control-Allow-Methods': 'OPTIONS, GET, POST', 'Access-Control-Allow-Credentials': 'true', } # use dict comprehension to filter headers to the ones we want to check self.assertEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected) # use invalid Origin f = requests.options(self.url_base() + '/rest/data/user', headers={ 'content-type': "application/json", 'x-requested-with': "rest", 'Access-Control-Request-Headers': "x-requested-with", 'Access-Control-Request-Method': "PUT", 'Origin': "ZZZ" }) self.assertEqual(f.status_code, 400) expected = '{ "error": { "status": 400, "msg": "Client is not ' \ 'allowed to use Rest Interface." } }' self.assertEqual(b2s(f.content), expected)
def nice_sender_header(name, address, charset): # construct an address header so it's as human-readable as possible # even in the presence of a non-ASCII name part if not name: return address try: encname = b2s(name.encode('ASCII')) except UnicodeEncodeError: # use Header to encode correctly. encname = Header(name, charset=charset).encode() # the important bits of formataddr() if specialsre.search(encname): encname = '"%s"' % escapesre.sub(r'\\\g<0>', encname) # now format the header as a string - don't return a Header as anonymous # headers play poorly with Messages (eg. won't get wrapped properly) return '%s <%s>' % (encname, address)
def h64encode(data): """encode using variant of base64""" return b2s(b64encode(data, b"./").strip(b"=\n"))
def pack_timestamp(): return b2s(base64.b64encode(struct.pack("i", int(time.time()))).strip())
def test_compression_gzip(self, method='gzip'): if method == 'gzip': decompressor = None elif method == 'br': decompressor = brotli.decompress elif method == 'zstd': decompressor = zstd.decompress # use basic auth for rest endpoint f = requests.get(self.url_base() + '/rest/data/user/1/username', auth=('admin', 'sekrit'), headers={ 'content-type': "", 'Accept-Encoding': '%s, foo' % method, 'Accept': '*/*' }) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'application/json', 'Access-Control-Allow-Credentials': 'true', 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', 'Content-Encoding': method, 'Vary': 'Origin, Accept-Encoding', } content_str = '''{ "data": { "id": "1", "link": "http://*****:*****@etag']) # type is "class 'str'" under py3, "type 'str'" py2 # just skip comparing it. del (json_dict['data']['type']) self.assertDictEqual(json_dict, content) # verify that ETag header ends with -<method> try: self.assertRegex(f.headers['ETag'], r'^"[0-9a-f]{32}-%s"$' % method) except AttributeError: # python2 no assertRegex so try substring match self.assertEqual(33, f.headers['ETag'].rindex('-' + method)) # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected) # use basic auth for rest endpoint, error case, bad attribute f = requests.get(self.url_base() + '/rest/data/user/1/foo', auth=('admin', 'sekrit'), headers={ 'content-type': "", 'Accept-Encoding': '%s, foo' % method, 'Accept': '*/*', 'Origin': 'ZZZZ' }) print(f.status_code) print(f.headers) # NOTE: not compressed payload too small self.assertEqual(f.status_code, 400) expected = { 'Content-Type': 'application/json', 'Access-Control-Allow-Credentials': 'true', 'Access-Control-Allow-Origin': 'ZZZZ', 'Allow': 'OPTIONS, GET, POST, PUT, DELETE, PATCH', 'Vary': 'Origin' } content = {"error": {"status": 400, "msg": "Invalid attribute foo"}} json_dict = json.loads(b2s(f.content)) self.assertDictEqual(json_dict, content) # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected) # test file x-fer f = requests.get(self.url_base() + '/@@file/user_utils.js', headers={ 'Accept-Encoding': '%s, foo' % method, 'Accept': '*/*' }) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'application/javascript', 'Content-Encoding': method, 'Vary': 'Accept-Encoding', } # compare to byte string as f.content may be compressed. # so running b2s on it will throw a UnicodeError if f.content[0:25] == b'// User Editing Utilities': # no need to decompress, urlib3.response did it for gzip and br data = f.content else: # I need to decode data = decompressor(f.content) # check first few bytes. self.assertEqual(b2s(data)[0:25], '// User Editing Utilities') # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected) # test file x-fer f = requests.get(self.url_base() + '/user1', headers={ 'Accept-Encoding': '%s, foo' % method, 'Accept': '*/*' }) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'text/html; charset=utf-8', 'Content-Encoding': method, 'Vary': 'Accept-Encoding', } if f.content[0:25] == b'<!-- dollarId: user.item,': # no need to decompress, urlib3.response did it for gzip and br data = f.content else: # I need to decode data = decompressor(f.content) # check first few bytes. self.assertEqual(b2s(data[0:25]), '<!-- dollarId: user.item,') # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected)
def test_compression_gzipfile(self): '''Get the compressed dummy file''' # create a user_utils.js.gz file to test pre-compressed # file serving code. Has custom contents to verify # that I get the compressed one. gzfile = "%s/html/user_utils.js.gzip" % self.dirname test_text = b"Custom text for user_utils.js\n" with gzip.open(gzfile, 'wb') as f: bytes_written = f.write(test_text) self.assertEqual(bytes_written, 30) # test file x-fer f = requests.get(self.url_base() + '/@@file/user_utils.js', headers={ 'Accept-Encoding': 'gzip, foo', 'Accept': '*/*' }) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'application/javascript', 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding', 'Content-Length': '69', } # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected) # check content - verify it's the .gz file not the real file. self.assertEqual(f.content, test_text) '''# verify that a different encoding request returns on the fly # test file x-fer using br, so we get runtime compression f = requests.get(self.url_base() + '/@@file/user_utils.js', headers = { 'Accept-Encoding': 'br, foo', 'Accept': '*/*'}) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'application/javascript', 'Content-Encoding': 'br', 'Vary': 'Accept-Encoding', 'Content-Length': '960', } # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual({ key: value for (key, value) in f.headers.items() if key in expected }, expected) try: from urllib3.response import BrotliDecoder # requests has decoded br to text for me data = f.content except ImportError: # I need to decode data = brotli.decompress(f.content) self.assertEqual(b2s(data)[0:25], '// User Editing Utilities') ''' # re-request file, but now make .gzip out of date. So we get the # real file compressed on the fly, not our test file. os.utime(gzfile, (0, 0)) # use 1970/01/01 or os base time f = requests.get(self.url_base() + '/@@file/user_utils.js', headers={ 'Accept-Encoding': 'gzip, foo', 'Accept': '*/*' }) print(f.status_code) print(f.headers) self.assertEqual(f.status_code, 200) expected = { 'Content-Type': 'application/javascript', 'Content-Encoding': 'gzip', 'Vary': 'Accept-Encoding', } # use dict comprehension to remove fields like date, # content-length etc. from f.headers. self.assertDictEqual( { key: value for (key, value) in f.headers.items() if key in expected }, expected) # check content - verify it's the real file, not crafted .gz. self.assertEqual(b2s(f.content)[0:25], '// User Editing Utilities') # cleanup os.remove(gzfile)
def send_message(self, issueid, msgid, note, sendto, from_address=None, bcc_sendto=[], subject=None, crypt=False, add_headers={}, authid=None): '''Actually send the nominated message from this issue to the sendto recipients, with the note appended. It's possible to add headers to the message with the add_headers variable. ''' users = self.db.user messages = self.db.msg files = self.db.file if msgid is None: inreplyto = None messageid = None else: inreplyto = messages.get(msgid, 'inreplyto') messageid = messages.get(msgid, 'messageid') # make up a messageid if there isn't one (web edit) if not messageid: # this is an old message that didn't get a messageid, so # create one messageid = "<%s.%s.%s%s@%s>" % ( time.time(), b2s(base64.b32encode(random_.token_bytes(10))), self.classname, issueid, self.db.config['MAIL_DOMAIN']) if msgid is not None: messages.set(msgid, messageid=messageid) # compose title cn = self.classname title = self.get(issueid, 'title') or '%s message copy' % cn # figure author information if authid: pass elif msgid: authid = messages.get(msgid, 'author') else: authid = self.db.getuid() authname = users.get(authid, 'realname') if not authname: authname = users.get(authid, 'username', '') authaddr = users.get(authid, 'address', '') if authaddr and self.db.config.MAIL_ADD_AUTHOREMAIL: authaddr = " <%s>" % formataddr(('', authaddr)) elif authaddr: authaddr = "" # make the message body m = [''] # put in roundup's signature if self.db.config.EMAIL_SIGNATURE_POSITION == 'top': m.append(self.email_signature(issueid, msgid)) # add author information if authid and self.db.config.MAIL_ADD_AUTHORINFO: if msgid and len(self.get(issueid, 'messages')) == 1: m.append( _("New submission from %(authname)s%(authaddr)s:") % locals()) elif msgid: m.append( _("%(authname)s%(authaddr)s added the comment:") % locals()) else: m.append(_("Change by %(authname)s%(authaddr)s:") % locals()) m.append('') # add the content if msgid is not None: m.append(messages.get(msgid, 'content', '')) # get the files for this message message_files = [] if msgid: for fileid in messages.get(msgid, 'files'): # check the attachment size filesize = self.db.filesize('file', fileid, None) if filesize <= self.db.config.NOSY_MAX_ATTACHMENT_SIZE: message_files.append(fileid) else: base = self.db.config.TRACKER_WEB link = "".join((base, files.classname, fileid)) filename = files.get(fileid, 'name') m.append( _("File '%(filename)s' not attached - " "you can download it from %(link)s.") % locals()) # add the change note if note: m.append(note) # put in roundup's signature if self.db.config.EMAIL_SIGNATURE_POSITION == 'bottom': m.append(self.email_signature(issueid, msgid)) # figure the encoding charset = getattr(self.db.config, 'EMAIL_CHARSET', 'utf-8') # construct the content and convert to unicode object body = s2u('\n'.join(m)) # make sure the To line is always the same (for testing mostly) sendto.sort() # make sure we have a from address if from_address is None: from_address = self.db.config.TRACKER_EMAIL # additional bit for after the From: "name" from_tag = getattr(self.db.config, 'EMAIL_FROM_TAG', '') if from_tag: from_tag = ' ' + from_tag if subject is None: subject = '[%s%s] %s' % (cn, issueid, title) author = (authname + from_tag, from_address) # send an individual message per recipient? if self.db.config.NOSY_EMAIL_SENDING != 'single': sendto = [[address] for address in sendto] else: sendto = [sendto] # tracker sender info tracker_name = s2u(self.db.config.TRACKER_NAME) tracker_name = nice_sender_header(tracker_name, from_address, charset) # now send one or more messages # TODO: I believe we have to create a new message each time as we # can't fiddle the recipients in the message ... worth testing # and/or fixing some day first = True for sendto in sendto: # create the message mailer = Mailer(self.db.config) message = mailer.get_standard_message(multipart=message_files) # set reply-to as requested by config option # TRACKER_REPLYTO_ADDRESS replyto_config = self.db.config.TRACKER_REPLYTO_ADDRESS if replyto_config: if replyto_config == "AUTHOR": # note that authaddr at this point is already # surrounded by < >, so get the original address # from the db as nice_send_header adds < > replyto_addr = nice_sender_header( authname, users.get(authid, 'address', ''), charset) else: replyto_addr = replyto_config else: replyto_addr = tracker_name message['Reply-To'] = replyto_addr # message ids if messageid: message['Message-Id'] = messageid if inreplyto: message['In-Reply-To'] = inreplyto # Generate a header for each link or multilink to # a class that has a name attribute for propname, prop in self.getprops().items(): if not isinstance(prop, (hyperdb.Link, hyperdb.Multilink)): continue cl = self.db.getclass(prop.classname) label = None if 'name' in cl.getprops(): label = 'name' if prop.msg_header_property in cl.getprops(): label = prop.msg_header_property if prop.msg_header_property == "": # if msg_header_property is set to empty string # suppress the header entirely. You can't use # 'msg_header_property == None'. None is the # default value. label = None if not label: continue if isinstance(prop, hyperdb.Link): value = self.get(issueid, propname) if value is None: continue values = [value] else: values = self.get(issueid, propname) if not values: continue values = [cl.get(v, label) for v in values] values = ', '.join(values) header = "X-Roundup-%s-%s" % (self.classname, propname) try: values.encode('ascii') message[header] = values except UnicodeError: message[header] = Header(values, charset) # Add header for main id number to make filtering # email easier than extracting from subject line. header = "X-Roundup-%s-Id" % (self.classname) values = issueid try: values.encode('ascii') message[header] = values except UnicodeError: message[header] = Header(values, charset) # Generate additional headers for k in add_headers: v = add_headers[k] try: v.encode('ascii') message[k] = v except UnicodeError: message[k] = Header(v, charset) if not inreplyto: # Default the reply to the first message msgs = self.get(issueid, 'messages') # Assume messages are sorted by increasing message number here # If the issue is just being created, and the submitter didn't # provide a message, then msgs will be empty. if msgs and msgs[0] != msgid: inreplyto = messages.get(msgs[0], 'messageid') if inreplyto: message['In-Reply-To'] = inreplyto # attach files if message_files: # first up the text as a part part = mailer.get_standard_message() part.set_payload(body, part.get_charset()) message.attach(part) for fileid in message_files: name = files.get(fileid, 'name') mime_type = (files.get(fileid, 'type') or mimetypes.guess_type(name)[0] or 'application/octet-stream') if mime_type == 'text/plain': content = files.get(fileid, 'content') part = MIMEText('') del part['Content-Transfer-Encoding'] try: enc = content.encode('ascii') part = mailer.get_text_message('us-ascii') part.set_payload(enc) except UnicodeError: # the content cannot be 7bit-encoded. # use quoted printable # XXX stuffed if we know the charset though :( part = mailer.get_text_message('utf-8') part.set_payload(content, part.get_charset()) elif mime_type == 'message/rfc822': content = files.get(fileid, 'content') main, sub = mime_type.split('/') p = FeedParser() p.feed(content) part = MIMEBase(main, sub) part.set_payload([p.close()]) else: # some other type, so encode it content = files.get(fileid, 'binary_content') main, sub = mime_type.split('/') part = MIMEBase(main, sub) part.set_payload(content) encoders.encode_base64(part) cd = 'Content-Disposition' part[cd] = 'attachment;\n filename="%s"' % name message.attach(part) else: message.set_payload(body, message.get_charset()) if crypt: send_msg = self.encrypt_to(message, sendto) else: send_msg = message mailer.set_message_attributes(send_msg, sendto, subject, author) if crypt: send_msg['Message-Id'] = message['Message-Id'] send_msg['Reply-To'] = message['Reply-To'] if message.get('In-Reply-To'): send_msg['In-Reply-To'] = message['In-Reply-To'] if sendto: mailer.smtp_send(sendto, send_msg.as_string()) if first: if crypt: # send individual bcc mails, otherwise receivers can # deduce bcc recipients from keys in message for bcc in bcc_sendto: send_msg = self.encrypt_to(message, [bcc]) send_msg['Message-Id'] = message['Message-Id'] send_msg['Reply-To'] = message['Reply-To'] if message.get('In-Reply-To'): send_msg['In-Reply-To'] = message['In-Reply-To'] mailer.smtp_send([bcc], send_msg.as_string()) elif bcc_sendto: mailer.smtp_send(bcc_sendto, send_msg.as_string()) first = False