def run(self):
"""Called by the threading framework to start the thread."""
first = self._port
last = self._port + self._retries
found = False
while self._port <= last:
# Passing the port parameter to web.py is ugly, but the mailing
# list entries I have found so far suggest this, and it does the
# job. A wrapper around sys would be the answer, which should be
# done anyway to control logging (there sys.stderr should be
# diverted).
sys.argv = (None, str(self._port),)
webapp = webpy.application(urls, globals())
self.webapp = webapp
try:
hkutils.log('Starting web service...')
webapp.run()
# I'm not sure this line will ever be executed
found = True
except socket.error, e:
# We don't want to catch exceptions that are not about an
# address already being in use
if re.search('Address already in use', str(e)):
self._port += 1
else:
exc_info = sys.exc_info()
raise exc_info[0], exc_info[1], exc_info[2]
def test__1(self):
"""Tests the following functions:
- :func:`hkutils.set_log`
- :func:`hkutils.log`
"""
# We use a custom log function
old_log_fun = hkutils.log_fun
def log_fun(*args):
log.append(list(args))
hkutils.set_log(log_fun)
# Test logging
log = []
hkutils.log('first line', 'second line')
hkutils.log('third line')
self.assertEqual(
log,
[['first line', 'second line'],
['third line']])
# Setting the original logging function back
hkutils.set_log(old_log_fun)
def write_all(self):
self.calc()
hkutils.log('Generating issues.html...')
self.options.html_title = 'Sorted issues'
filename = 'index/issues-%s.html' % (self._heap_id,)
self.write_page(
filename,
self.print_issues_page())
def edit_files(files):
"""DEPRECATED. Opens an editor in which the user edits the given files.
Please use :func:`hkshell.edit_files` instead.
It invokes the editor program stored in the ``EDITOR`` environment
variable. If ``EDITOR`` is undefined or empty, it invokes the default
editor on the system using the :func:`default_editor` function.
**Type:** |EditFileFun|
"""
old_content = {}
for file in files:
old_content[file] = hkutils.file_to_string(file, return_none=True)
editor = os.getenv('HEAPKEEPER_EDITOR')
# if HEAPKEEPER_EDITOR is not set, get EDITOR
if editor is None or editor == '':
editor = os.getenv('EDITOR')
# if EDITOR is not set, get the default editor
if editor is None or editor == '':
editor = default_editor()
# if not even the default is set, print an error message
if editor is None:
hkutils.log(
'Cannot determine the default editor based on the operating\n'
'system. Please set the EDITOR environment variable to the editor\n'
'you want to use or set hkshell.options.callback.edit_files to\n'
'call your editor of choice.')
return False
try:
editor = editor_to_editor_list(editor)
except IncorrectEditorException:
hkutils.log(
'The editor variable is incorrect:\n' +
editor +
'Please set the EDITOR environment variable to the editor\n'
'you want to use or set hkshell.options.callback.edit_files to\n'
'call your editor of choice.')
return False
subprocess.call(editor + files)
def did_file_change(file):
new_content = hkutils.file_to_string(file, return_none=True)
return old_content[file] != new_content
changed_files = filter(did_file_change, files)
return changed_files
def __init__(self, postdb):
"""Constructor.
**Arguments:**
- `postdb` (|PostDB|)
"""
hkutils.log(
"WARNING: This class (hkgen.Generator) is deprecated.\n" "Use BaseGenerator or StaticGenerator instead."
)
StaticGenerator.__init__(self, postdb)
def last():
"""Display the time of the last access in a conveniently interpretable
("human-readable") format."""
access_datetimes = last_access.values()
if len(access_datetimes) == 0:
hkutils.log("Access list is empty.")
return
access_datetimes.sort()
last_datetime = access_datetimes[-1]
now_datetime = datetime.datetime.now()
last_str = hkutils.humanize_timedelta(now_datetime - last_datetime)
hkutils.log("Last access was %s ago." % (last_str,))
def create_post_from_email(self, header, text):
"""Create a Post from an already downloaded email header and
body part.
**Arguments:**
- `header` (str) -- The header of the email.
- `text` (str) -- The body of the email.
**Returns:** (str, str)
"""
headers, text = self.parse_email(header, text)
post = hklib.Post.create_empty()
post.set_author(headers.get('From', ''))
post.set_subject(headers.get('Subject', ''))
post.set_messid(headers.get('Message-Id', ''))
post.set_parent(headers.get('In-Reply-To', ''))
post.set_date(headers.get('Date', ''))
post.set_body(text)
post.remove_google_stuff()
post.remove_newlines_from_subject()
post.normalize_subject()
# If the author has a nickname, we set it
r = re.compile('[-._A-Za-z0-9]+@[-._A-Za-z0-9]+')
match = r.search(post.author())
if match is not None:
author_address = match.group(0)
else:
author_address = ''
# We try to read the nickname from the
# heaps/<heap id>/nicknames/<author address> configuration item
heap_config = self._config['heaps'][self._heap_id]
author_nick = heap_config['nicknames'].get(author_address)
# We try to read the nickname from the nicknames/<author address>
# configuration item
if author_nick is None:
author_nick = self._config['nicknames'].get(author_address)
if author_nick is not None:
post.set_author(author_nick)
else:
hkutils.log("WARNING: author's nickname not found: %s" %
(post.author()))
return post
def set_to_reviewed(prepost=None):
"""Sets the given thread to reviewed and commits all changes to the heap in
which the thread is.
**Argument:**
- `prepost` (|PrePost| | ``None``) -- If all touched post are within one
thread, this parameter may be ``None``, in which case Heapkeeper will
find this thread.
"""
if prepost is None:
modified = hkshell.postdb().all().collect.is_modified()
roots = modified.expb().collect.is_root()
if len(roots) == 0:
hkutils.log(
'No modified posts. No action. Use the r(<post id>) format.')
return
elif len(roots) > 1:
hkutils.log('More than one modified threads. No action.')
return
post = roots.pop()
hkutils.log('Thread: ' + post.post_id_str())
else:
post = hkshell.p(prepost)
if hkshell.postdb().parent(post) is not None:
hkutils.log('The post is not a root. No action.')
return
# Saving the modifications
hkshell.aTr(post, 'reviewed')
hkshell.s()
# Calculating the subject to be mentioned in the commit message
subject = post.subject()
if len(subject) > 33:
subject = subject[:30] + '...'
# Writing the commit message into a temp file
f, filename = tempfile.mkstemp()
os.write(f, 'Review: "' + subject + '" thread\n'
'\n'
'[review]\n')
os.close(f)
# Commiting the change in git
heap_dir = hkshell.postdb()._heaps[post.heap_id()]
oldcwd = os.getcwd()
try:
os.chdir(heap_dir)
hkutils.call(['git', 'commit', '-aF', filename])
finally:
os.chdir(oldcwd)
os.remove(filename)
def write_thread_pages(self, write_all=False):
"""Writes the thread pages into
``'<root_heap_id>/thread_<root_post_index>.html'``.
**Argument:**
- `write_all` (bool) -- If ``True``, the function writes all thread
pages. Otherwise it writes only those whose posts were modified.
"""
hkutils.log("Generating thread pages...")
if write_all:
posts = self._postdb.roots()
else:
posts = self.outdated_thread_pages()
for post in posts:
self.options.html_title = post.subject()
self.write_page(filename=post.htmlthreadbasename(), html_body=self.print_thread_page(post))
def connect(self):
"""Connects to the IMAP server."""
hkutils.log('Reading settings...')
# Try to get the server configuration specific to the heap
server = self._config['heaps'][self._heap_id].get('server')
# If that does not exist, get the generic server configuration
if server is None:
server = self._config['server']
host = server['host']
port = server['port']
username = server['username']
password = server.get('password')
# if imaps is omitted, default to True iff port is 993
imaps = server.get('imaps', (port == 993))
# If no password was specified, ask it from the user
if password is None:
prompt = 'Password for %s@%s: ' % (username, host)
password = getpass.getpass()
hkutils.log('Connecting...')
if imaps:
self._server = imaplib.IMAP4_SSL(host, port)
else:
self._server = imaplib.IMAP4(host, port)
self._server.login(username, password)
hkutils.log('Connected')
def settle_files_to_copy(self):
"""Copies the files in `self.options.files_to_copy` to the HTML
directory."""
for file in self.options.files_to_copy:
target_file = os.path.join(self._postdb.html_dir(), file)
# We create the target directory if necessary
target_dir = os.path.dirname(target_file)
if not os.path.exists(target_dir):
os.makedirs(target_dir)
# We try to copy the file from one of the heaps used by the post
# database
# We have not found the file yet
file_found = False
# We iterate the heaps in alphabetical order (to make the algorithm
# deterministic)
heaps = sorted(self._postdb._heaps.items())
# We iterate the heaps and look for the file
for heap_id, heap_dir in heaps:
heap_file = os.path.join(heap_dir, file)
if os.path.exists(heap_file):
shutil.copyfile(heap_file, target_file)
file_found = True
break
if file_found:
# File already found and copied
pass
elif os.path.exists(file):
# Copy from the current directory
shutil.copyfile(file, target_file)
else:
hkutils.log('WARNING: file "%s" not found' % (file,))
def default_deny(realm, username, password, redirect_url):
# Unused arguments 'password', 'realm', 'redirect_url'
# pylint: disable=W0613
hkutils.log('Access denied for user %s.' % username)
return 'Authentication needed!'
for testmodule in collect_testmodules_from_dir(src_dir)]
def main(args):
hkutils.set_log(False)
if len(args) > 0 and args[0] in ['-h', '--help']:
sys.stdout.write(__doc__)
elif args == []:
testmodules = collect_testmodules()
suites = \
[ unittest.TestLoader().loadTestsFromModule(__import__(modname))
for modname in testmodules ]
suite = unittest.TestSuite(suites)
unittest.TextTestRunner(verbosity=0).run(suite)
elif len(args) == 3:
modname, testcasename, funname = args
suite = unittest.TestSuite()
testcase = getattr(__import__(modname), testcasename)
suite.addTest(testcase(funname))
unittest.TextTestRunner(verbosity=0).run(suite)
else:
print 'Incorrect arguments.\n' \
'Run "python test.py --help" for help.'
if __name__ == '__main__':
if not os.path.isfile(os.path.join('src', 'test.py')):
hkutils.log('Error: You are not in the Heapkeeper main directory.')
sys.exit(1)
main(sys.argv[1:])
def download_new(self, lower_value=0, detailed_log=False):
"""Downloads the new emails from the INBOX of the IMAP server and adds
them to the post database.
**Arguments:**
- `lower_value` (int) -- Only the email indices that are greater or
equal to `lower_value` are examined.
- `detailed_log` (bool) -- If ``True``, every email found or downloaded
is reported.
**Returns:** |PostSet| -- A set of the new posts.
"""
assert self._postdb.has_heap_id(self._heap_id)
self._server.select("INBOX")
# Stage 1: checking for matching IDs
# Note: this is probably unnecessary
if lower_value == 0:
imap_crit = 'ALL'
else:
imap_crit = '(%d:*)' % (lower_value,)
result = self._server.search(None, imap_crit)[1][0].strip()
if result == '':
hkutils.log('No messages to download.')
return
emails = result.split(' ')
# The list has to be compacted to keep the IMAP command short.
# Plus we fragment the list into pieces not longer than
# a given amount that can be safely handled by all IMAP servers
# (this is now 500).
sequences = self.imap_compact(emails)
fragments = self.fragment_list(sequences)
# Stage 2: checking which messages we don't already have
# We do this because it can save a lot of time and bandwidth
hkutils.log('Checking...')
messids = []
for fragment in fragments:
fragment_imap = ','.join(fragment)
param = '(BODY[HEADER.FIELDS (MESSAGE-ID)])'
result = self._server.fetch(fragment_imap, param)[1]
raw_messids = [result[2 * i][1] for i in range(len(result) / 2)]
# using the email lib to parse Message-Id headers
messids.extend([email.message_from_string(s)['Message-Id']
for s in raw_messids])
# Assembling a list of new messages
download_list = []
for index, messid in zip(emails, messids):
post = self._postdb.post_by_messid(messid)
if post == None:
download_list.append(index)
elif detailed_log:
hkutils.log('Post #%s (#%s in INBOX) found.' %
(post.post_index(), int(index) + lower_value))
# Stage 3: actually downloading the needed messages
new_msg_count = len(download_list)
if new_msg_count == 0:
hkutils.log('No new messages.')
return
else:
hkutils.log('%d new message%s found.' %
(new_msg_count, hkutils.plural(new_msg_count)))
hkutils.log('Downloading...')
# Compact & fragment this list too to keep IMAP commands short
sequences = self.imap_compact(download_list)
fragments = self.fragment_list(sequences)
downloaded_msg_count = 0
for fragment in fragments:
fragment_imap = ','.join(fragment)
new_posts = []
result = self._server.fetch(fragment_imap,
'(BODY[TEXT] BODY[HEADER])')[1]
# `result` is formatted like this:
#
# [ # First email:
#
# # result[0]:
# ('1 (BODY[TEXT] {123}',
# 'This is the body of the first result...'),
#
# # result[1]:
# (' BODY[HEADER] {456}',
# 'Headers-Of: first-email...'),
#
# # result[2]:
# ')'
#
# # Second email:
#
# # result[3]:
# ('2 (BODY[TEXT] {789}',
# 'This is the body of the second result...'),
# ... ]
#
# The final ')' element causes a single email to be represented
# by 3 elements in the list of results.
for i in range(len(result) / 3):
number_str = result[i * 3][0]
number = number_str[0:number_str.index(' ')]
text = result[i * 3][1]
header = result[i * 3 + 1][1]
# Catch "Exception" # pylint: disable=W0703
try:
post = self.create_post_from_email(header, text)
self._postdb.add_new_post(post, self._heap_id)
new_posts.append(post)
except Exception, e:
hkutils.log('Error while downloading: %s' % e)
downloaded_msg_count += 1
continue
if detailed_log:
hkutils.log('Post #%s (#%s in INBOX) downloaded.' %
(post.post_index(), number))
downloaded_msg_count += 1
def parse_email(header, text):
"""Creates strings from an already downloaded email header and
body part.
**Arguments:**
- `header` (str) -- The header of the email.
- `text` (str) -- The body of the email.
**Returns:** (str, str)
"""
message = email.message_from_string(header + text)
def normalize_str(s):
s = re.sub(r'\r\n', r'\n', s) # Windows EOL
s = re.sub(r'\xc2\xa0', ' ', s) # Non-breaking space
return s
# processing the header
headers = {}
for attr in ['From', 'Subject', 'Message-Id', 'In-Reply-To', 'Date']:
value = message[attr]
if value != None:
# valuelist::[(string, encoding)]
valuelist = email.header.decode_header(value)
value = ''
first = True
for v in valuelist:
if first:
first = False
else:
value += ' '
value += hkutils.utf8(v[0], v[1])
value = normalize_str(value)
headers[attr] = value
# We find the first leaf of the "message tree", where the multipart
# messages are the branches and the non-multipart messages are the
# leaves. This leaf should has the content type text/plain. When the
# loop is finished, the `message` variable will point to the leaf that
# is interesting to Heapkeeper.
#
# See also http://en.wikipedia.org/wiki/MIME#Multipart_messages
while True:
content_type = message.get_content_type()
if message.is_multipart():
if (content_type not in
('multipart/mixed',
'multipart/alternative',
'multipart/signed',
'multipart/related')):
hkutils.log('WARNING: unknown type of multipart '
'message: %s\n%s' %
(content_type, header + text))
message = message.get_payload(0)
else:
if content_type != 'text/plain':
hkutils.log('WARNING: content type is not text/plain: '
'%s\n%s' %
(content_type, header + text))
break
# encoding
encoding = message['Content-Transfer-Encoding']
text = message.get_payload()
if type(text) is not str:
raise hkutils.HkException(
'`text` is not a string but the following object: %s\n' %
(str(text),))
if encoding != None:
if encoding.lower() in ('7bit', '8bit', 'binary'):
pass # no conversion needed
elif encoding.lower() == 'base64':
text = base64.b64decode(text)
elif encoding.lower() == 'quoted-printable':
text = quopri.decodestring(text)
else:
hkutils.log('WARNING: Unknown encoding, skipping decoding: '
'%s\n'
'text:\n%s\n' % (encoding, text))
charset = message.get_content_charset()
text = hkutils.utf8(text, charset)
text = normalize_str(text)
return headers, text
for i in range(len(result) / 3):
number_str = result[i * 3][0]
number = number_str[0:number_str.index(' ')]
text = result[i * 3][1]
header = result[i * 3 + 1][1]
# Catch "Exception" # pylint: disable=W0703
try:
post = self.create_post_from_email(header, text)
self._postdb.add_new_post(post, self._heap_id)
new_posts.append(post)
except Exception, e:
hkutils.log('Error while downloading: %s' % e)
downloaded_msg_count += 1
continue
if detailed_log:
hkutils.log('Post #%s (#%s in INBOX) downloaded.' %
(post.post_index(), number))
downloaded_msg_count += 1
hkutils.log('%d new message%s downloaded.' %
(downloaded_msg_count,
hkutils.plural(downloaded_msg_count)))
if new_msg_count != downloaded_msg_count:
hkutils.log('WARNING: number of new (%d) and'
'downloaded (%d) messages not equal!' %
(new_msg_count, downloaded_msg_count))
return hklib.PostSet(self._postdb, new_posts)
def write_main_index_page(self):
"""Writes the main index page into ``'index.html'``."""
hkutils.log("Generating index.html...")
self.options.html_title = "Main index"
self.write_page("index/index.html", self.print_main_index_page())
def write_given_posts_page(self):
"""Writes the "given post" page."""
hkutils.log("Generating %s..." % (self.html_filename,))
self.write_page(self.html_filename, self.print_given_posts_page())