예제 #1
0
    def run(self):
        """Called by the threading framework to start the thread."""

        first = self._port
        last = self._port + self._retries
        found = False

        while self._port <= last:
            # Passing the port parameter to web.py is ugly, but the mailing
            # list entries I have found so far suggest this, and it does the
            # job. A wrapper around sys would be the answer, which should be
            # done anyway to control logging (there sys.stderr should be
            # diverted).
            sys.argv = (None, str(self._port),)
            webapp = webpy.application(urls, globals())
            self.webapp = webapp

            try:
                hkutils.log('Starting web service...')
                webapp.run()

                # I'm not sure this line will ever be executed
                found = True

            except socket.error, e:
                # We don't want to catch exceptions that are not about an
                # address already being in use
                if re.search('Address already in use', str(e)):
                    self._port += 1
                else:
                    exc_info = sys.exc_info()
                    raise exc_info[0], exc_info[1], exc_info[2]
예제 #2
0
    def test__1(self):

        """Tests the following functions:

        - :func:`hkutils.set_log`
        - :func:`hkutils.log`
        """

        # We use a custom log function
        old_log_fun = hkutils.log_fun
        def log_fun(*args):
            log.append(list(args))
        hkutils.set_log(log_fun)

        # Test logging
        log = []
        hkutils.log('first line', 'second line')
        hkutils.log('third line')
        self.assertEqual(
            log,
            [['first line', 'second line'],
             ['third line']])

        # Setting the original logging function back
        hkutils.set_log(old_log_fun)
예제 #3
0
 def write_all(self):
     self.calc()
     hkutils.log('Generating issues.html...')
     self.options.html_title = 'Sorted issues'
     filename = 'index/issues-%s.html' % (self._heap_id,)
     self.write_page(
         filename,
         self.print_issues_page())
예제 #4
0
def edit_files(files):
    """DEPRECATED. Opens an editor in which the user edits the given files.

    Please use :func:`hkshell.edit_files` instead.

    It invokes the editor program stored in the ``EDITOR`` environment
    variable. If ``EDITOR`` is undefined or empty, it invokes the default
    editor on the system using the :func:`default_editor` function.

    **Type:** |EditFileFun|
    """

    old_content = {}
    for file in files:
        old_content[file] = hkutils.file_to_string(file, return_none=True)

    editor = os.getenv('HEAPKEEPER_EDITOR')

    # if HEAPKEEPER_EDITOR is not set, get EDITOR
    if editor is None or editor == '':
        editor = os.getenv('EDITOR')

    # if EDITOR is not set, get the default editor
    if editor is None or editor == '':
        editor = default_editor()

    # if not even the default is set, print an error message
    if editor is None:
        hkutils.log(
           'Cannot determine the default editor based on the operating\n'
           'system. Please set the EDITOR environment variable to the editor\n'
           'you want to use or set hkshell.options.callback.edit_files to\n'
           'call your editor of choice.')
        return False

    try:
        editor = editor_to_editor_list(editor)
    except IncorrectEditorException:
        hkutils.log(
            'The editor variable is incorrect:\n' +
            editor +
            'Please set the EDITOR environment variable to the editor\n'
            'you want to use or set hkshell.options.callback.edit_files to\n'
            'call your editor of choice.')
        return False

    subprocess.call(editor + files)

    def did_file_change(file):
        new_content = hkutils.file_to_string(file, return_none=True)
        return old_content[file] != new_content

    changed_files = filter(did_file_change, files)
    return changed_files
예제 #5
0
    def __init__(self, postdb):
        """Constructor.

        **Arguments:**

        - `postdb` (|PostDB|)
        """

        hkutils.log(
            "WARNING: This class (hkgen.Generator) is deprecated.\n" "Use BaseGenerator or StaticGenerator instead."
        )
        StaticGenerator.__init__(self, postdb)
예제 #6
0
def last():
    """Display the time of the last access in a conveniently interpretable
    ("human-readable") format."""

    access_datetimes = last_access.values()
    if len(access_datetimes) == 0:
        hkutils.log("Access list is empty.")
        return
    access_datetimes.sort()
    last_datetime = access_datetimes[-1]
    now_datetime = datetime.datetime.now()
    last_str = hkutils.humanize_timedelta(now_datetime - last_datetime)
    hkutils.log("Last access was %s ago." % (last_str,))
예제 #7
0
    def create_post_from_email(self, header, text):
        """Create a Post from an already downloaded email header and
        body part.

        **Arguments:**

        - `header` (str) -- The header of the email.
        - `text` (str) -- The body of the email.

        **Returns:** (str, str)
        """

        headers, text = self.parse_email(header, text)
        post = hklib.Post.create_empty()
        post.set_author(headers.get('From', ''))
        post.set_subject(headers.get('Subject', ''))
        post.set_messid(headers.get('Message-Id', ''))
        post.set_parent(headers.get('In-Reply-To', ''))
        post.set_date(headers.get('Date', ''))
        post.set_body(text)
        post.remove_google_stuff()
        post.remove_newlines_from_subject()
        post.normalize_subject()

        # If the author has a nickname, we set it
        r = re.compile('[-._A-Za-z0-9]+@[-._A-Za-z0-9]+')
        match = r.search(post.author())
        if match is not None:
            author_address = match.group(0)
        else:
            author_address = ''

        # We try to read the nickname from the
        # heaps/<heap id>/nicknames/<author address> configuration item
        heap_config = self._config['heaps'][self._heap_id]
        author_nick = heap_config['nicknames'].get(author_address)

        # We try to read the nickname from the nicknames/<author address>
        # configuration item
        if author_nick is None:
            author_nick = self._config['nicknames'].get(author_address)

        if author_nick is not None:
            post.set_author(author_nick)
        else:
            hkutils.log("WARNING: author's nickname not found: %s" %
                        (post.author()))

        return post
예제 #8
0
def set_to_reviewed(prepost=None):
    """Sets the given thread to reviewed and commits all changes to the heap in
    which the thread is.

    **Argument:**

    - `prepost` (|PrePost| | ``None``) -- If all touched post are within one
      thread, this parameter may be ``None``, in which case Heapkeeper will
      find this thread.
    """

    if prepost is None:
        modified = hkshell.postdb().all().collect.is_modified()
        roots = modified.expb().collect.is_root()
        if len(roots) == 0:
            hkutils.log(
                'No modified posts. No action. Use the r(<post id>) format.')
            return
        elif len(roots) > 1:
            hkutils.log('More than one modified threads. No action.')
            return

        post = roots.pop()
        hkutils.log('Thread: ' + post.post_id_str())
    else:
        post = hkshell.p(prepost)
        if hkshell.postdb().parent(post) is not None:
            hkutils.log('The post is not a root. No action.')
            return

    # Saving the modifications
    hkshell.aTr(post, 'reviewed')
    hkshell.s()

    # Calculating the subject to be mentioned in the commit message
    subject = post.subject()
    if len(subject) > 33:
        subject = subject[:30] + '...'

    # Writing the commit message into a temp file
    f, filename = tempfile.mkstemp()
    os.write(f, 'Review: "' + subject + '" thread\n'
                '\n'
                '[review]\n')
    os.close(f)

    # Commiting the change in git
    heap_dir = hkshell.postdb()._heaps[post.heap_id()]
    oldcwd = os.getcwd()
    try:
        os.chdir(heap_dir)
        hkutils.call(['git', 'commit', '-aF', filename])
    finally:
        os.chdir(oldcwd)
        os.remove(filename)
예제 #9
0
    def write_thread_pages(self, write_all=False):
        """Writes the thread pages into
        ``'<root_heap_id>/thread_<root_post_index>.html'``.

        **Argument:**

        - `write_all` (bool) -- If ``True``, the function writes all thread
          pages. Otherwise it writes only those whose posts were modified.
        """

        hkutils.log("Generating thread pages...")

        if write_all:
            posts = self._postdb.roots()
        else:
            posts = self.outdated_thread_pages()

        for post in posts:
            self.options.html_title = post.subject()
            self.write_page(filename=post.htmlthreadbasename(), html_body=self.print_thread_page(post))
예제 #10
0
    def connect(self):
        """Connects to the IMAP server."""

        hkutils.log('Reading settings...')

        # Try to get the server configuration specific to the heap
        server = self._config['heaps'][self._heap_id].get('server')

        # If that does not exist, get the generic server configuration
        if server is None:
            server = self._config['server']

        host = server['host']
        port = server['port']
        username = server['username']
        password = server.get('password')
        # if imaps is omitted, default to True iff port is 993
        imaps = server.get('imaps', (port == 993))

        # If no password was specified, ask it from the user
        if password is None:
            prompt = 'Password for %s@%s: ' % (username, host)
            password = getpass.getpass()

        hkutils.log('Connecting...')
        if imaps:
            self._server = imaplib.IMAP4_SSL(host, port)
        else:
            self._server = imaplib.IMAP4(host, port)
        self._server.login(username, password)
        hkutils.log('Connected')
예제 #11
0
    def settle_files_to_copy(self):
        """Copies the files in `self.options.files_to_copy` to the HTML
        directory."""

        for file in self.options.files_to_copy:

            target_file = os.path.join(self._postdb.html_dir(), file)

            # We create the target directory if necessary
            target_dir = os.path.dirname(target_file)
            if not os.path.exists(target_dir):
                os.makedirs(target_dir)

            # We try to copy the file from one of the heaps used by the post
            # database

            # We have not found the file yet
            file_found = False

            # We iterate the heaps in alphabetical order (to make the algorithm
            # deterministic)
            heaps = sorted(self._postdb._heaps.items())

            # We iterate the heaps and look for the file
            for heap_id, heap_dir in heaps:
                heap_file = os.path.join(heap_dir, file)
                if os.path.exists(heap_file):
                    shutil.copyfile(heap_file, target_file)
                    file_found = True
                    break

            if file_found:
                # File already found and copied
                pass
            elif os.path.exists(file):
                # Copy from the current directory
                shutil.copyfile(file, target_file)
            else:
                hkutils.log('WARNING: file "%s" not found' % (file,))
예제 #12
0
def default_deny(realm, username, password, redirect_url):
    # Unused arguments 'password', 'realm', 'redirect_url'
    # pylint: disable=W0613
    hkutils.log('Access denied for user %s.' % username)
    return 'Authentication needed!'
예제 #13
0
            for testmodule in collect_testmodules_from_dir(src_dir)]


def main(args):

    hkutils.set_log(False)
    if len(args) > 0 and args[0] in ['-h', '--help']:
        sys.stdout.write(__doc__)
    elif args == []:
        testmodules = collect_testmodules()
        suites = \
            [ unittest.TestLoader().loadTestsFromModule(__import__(modname))
              for modname in testmodules ]
        suite = unittest.TestSuite(suites)
        unittest.TextTestRunner(verbosity=0).run(suite)
    elif len(args) == 3:
        modname, testcasename, funname = args
        suite = unittest.TestSuite()
        testcase = getattr(__import__(modname), testcasename)
        suite.addTest(testcase(funname))
        unittest.TextTestRunner(verbosity=0).run(suite)
    else:
        print 'Incorrect arguments.\n' \
              'Run "python test.py --help" for help.'

if __name__ == '__main__':
    if not os.path.isfile(os.path.join('src', 'test.py')):
        hkutils.log('Error: You are not in the Heapkeeper main directory.')
        sys.exit(1)
    main(sys.argv[1:])
예제 #14
0
    def download_new(self, lower_value=0, detailed_log=False):
        """Downloads the new emails from the INBOX of the IMAP server and adds
        them to the post database.

        **Arguments:**

        - `lower_value` (int) -- Only the email indices that are greater or
          equal to `lower_value` are examined.
        - `detailed_log` (bool) -- If ``True``, every email found or downloaded
          is reported.

        **Returns:** |PostSet| -- A set of the new posts.
        """

        assert self._postdb.has_heap_id(self._heap_id)
        self._server.select("INBOX")

        # Stage 1: checking for matching IDs
        # Note: this is probably unnecessary

        if lower_value == 0:
            imap_crit = 'ALL'
        else:
            imap_crit = '(%d:*)' % (lower_value,)
        result = self._server.search(None, imap_crit)[1][0].strip()
        if result == '':
            hkutils.log('No messages to download.')
            return
        emails = result.split(' ')

        # The list has to be compacted to keep the IMAP command short.
        # Plus we fragment the list into pieces not longer than
        # a given amount that can be safely handled by all IMAP servers
        # (this is now 500).
        sequences = self.imap_compact(emails)
        fragments = self.fragment_list(sequences)

        # Stage 2: checking which messages we don't already have
        # We do this because it can save a lot of time and bandwidth

        hkutils.log('Checking...')

        messids = []
        for fragment in fragments:
            fragment_imap = ','.join(fragment)
            param = '(BODY[HEADER.FIELDS (MESSAGE-ID)])'
            result = self._server.fetch(fragment_imap, param)[1]
            raw_messids = [result[2 * i][1] for i in range(len(result) / 2)]
            # using the email lib to parse Message-Id headers
            messids.extend([email.message_from_string(s)['Message-Id']
                            for s in raw_messids])

        # Assembling a list of new messages
        download_list = []
        for index, messid in zip(emails, messids):
            post = self._postdb.post_by_messid(messid)
            if post == None:
                download_list.append(index)
            elif detailed_log:
                hkutils.log('Post #%s (#%s in INBOX) found.' %
                            (post.post_index(), int(index) + lower_value))

        # Stage 3: actually downloading the needed messages

        new_msg_count = len(download_list)
        if new_msg_count == 0:
            hkutils.log('No new messages.')
            return
        else:
            hkutils.log('%d new message%s found.' %
                        (new_msg_count, hkutils.plural(new_msg_count)))

        hkutils.log('Downloading...')

        # Compact & fragment this list too to keep IMAP commands short
        sequences = self.imap_compact(download_list)
        fragments = self.fragment_list(sequences)

        downloaded_msg_count = 0
        for fragment in fragments:
            fragment_imap = ','.join(fragment)
            new_posts = []

            result = self._server.fetch(fragment_imap,
                                        '(BODY[TEXT] BODY[HEADER])')[1]
            # `result` is formatted like this:
            #
            #    [ # First email:
            #
            #      # result[0]:
            #      ('1 (BODY[TEXT] {123}',
            #       'This is the body of the first result...'),
            #
            #      # result[1]:
            #      (' BODY[HEADER] {456}',
            #       'Headers-Of: first-email...'),
            #
            #      # result[2]:
            #      ')'
            #
            #      # Second email:
            #
            #      # result[3]:
            #      ('2 (BODY[TEXT] {789}',
            #       'This is the body of the second result...'),
            #      ... ]
            #
            # The final ')' element causes a single email to be represented
            # by 3 elements in the list of results.

            for i in range(len(result) / 3):
                number_str = result[i * 3][0]
                number = number_str[0:number_str.index(' ')]
                text = result[i * 3][1]
                header = result[i * 3 + 1][1]
                # Catch "Exception" # pylint: disable=W0703
                try:
                    post = self.create_post_from_email(header, text)
                    self._postdb.add_new_post(post, self._heap_id)
                    new_posts.append(post)
                except Exception, e:
                    hkutils.log('Error while downloading: %s' % e)
                    downloaded_msg_count += 1
                    continue
                if detailed_log:
                    hkutils.log('Post #%s (#%s in INBOX) downloaded.' %
                                (post.post_index(), number))
                downloaded_msg_count += 1
예제 #15
0
    def parse_email(header, text):
        """Creates strings from an already downloaded email header and
        body part.

        **Arguments:**

        - `header` (str) -- The header of the email.
        - `text` (str) -- The body of the email.

        **Returns:** (str, str)
        """

        message = email.message_from_string(header + text)

        def normalize_str(s):
            s = re.sub(r'\r\n', r'\n', s) # Windows EOL
            s = re.sub(r'\xc2\xa0', ' ', s) # Non-breaking space
            return s

        # processing the header
        headers = {}
        for attr in ['From', 'Subject', 'Message-Id', 'In-Reply-To', 'Date']:
            value = message[attr]
            if value != None:
                # valuelist::[(string, encoding)]
                valuelist = email.header.decode_header(value)
                value = ''
                first = True
                for v in valuelist:
                    if first:
                        first = False
                    else:
                        value += ' '
                    value += hkutils.utf8(v[0], v[1])
                value = normalize_str(value)
                headers[attr] = value

        # We find the first leaf of the "message tree", where the multipart
        # messages are the branches and the non-multipart messages are the
        # leaves. This leaf should has the content type text/plain. When the
        # loop is finished, the `message` variable will point to the leaf that
        # is interesting to Heapkeeper.
        #
        # See also http://en.wikipedia.org/wiki/MIME#Multipart_messages
        while True:
            content_type = message.get_content_type()
            if message.is_multipart():
                if (content_type not in
                    ('multipart/mixed',
                     'multipart/alternative',
                     'multipart/signed',
                     'multipart/related')):
                    hkutils.log('WARNING: unknown type of multipart '
                                'message: %s\n%s' %
                                (content_type, header + text))
                message = message.get_payload(0)
            else:
                if content_type != 'text/plain':
                    hkutils.log('WARNING: content type is not text/plain: '
                                '%s\n%s' %
                                (content_type, header + text))
                break

        # encoding
        encoding = message['Content-Transfer-Encoding']
        text = message.get_payload()
        if type(text) is not str:
            raise hkutils.HkException(
                '`text` is not a string but the following object: %s\n' %
                (str(text),))
        if encoding != None:
            if encoding.lower() in ('7bit', '8bit', 'binary'):
                pass # no conversion needed
            elif encoding.lower() == 'base64':
                text = base64.b64decode(text)
            elif encoding.lower() == 'quoted-printable':
                text = quopri.decodestring(text)
            else:
                hkutils.log('WARNING: Unknown encoding, skipping decoding: '
                            '%s\n'
                            'text:\n%s\n' % (encoding, text))
        charset = message.get_content_charset()
        text = hkutils.utf8(text, charset)
        text = normalize_str(text)

        return headers, text
예제 #16
0
            for i in range(len(result) / 3):
                number_str = result[i * 3][0]
                number = number_str[0:number_str.index(' ')]
                text = result[i * 3][1]
                header = result[i * 3 + 1][1]
                # Catch "Exception" # pylint: disable=W0703
                try:
                    post = self.create_post_from_email(header, text)
                    self._postdb.add_new_post(post, self._heap_id)
                    new_posts.append(post)
                except Exception, e:
                    hkutils.log('Error while downloading: %s' % e)
                    downloaded_msg_count += 1
                    continue
                if detailed_log:
                    hkutils.log('Post #%s (#%s in INBOX) downloaded.' %
                                (post.post_index(), number))
                downloaded_msg_count += 1

        hkutils.log('%d new message%s downloaded.' %
                    (downloaded_msg_count,
                     hkutils.plural(downloaded_msg_count)))

        if new_msg_count != downloaded_msg_count:
            hkutils.log('WARNING: number of new (%d) and'
                        'downloaded (%d) messages not equal!' %
                        (new_msg_count, downloaded_msg_count))

        return hklib.PostSet(self._postdb, new_posts)
예제 #17
0
    def write_main_index_page(self):
        """Writes the main index page into ``'index.html'``."""

        hkutils.log("Generating index.html...")
        self.options.html_title = "Main index"
        self.write_page("index/index.html", self.print_main_index_page())
예제 #18
0
 def write_given_posts_page(self):
     """Writes the "given post" page."""
     hkutils.log("Generating %s..." % (self.html_filename,))
     self.write_page(self.html_filename, self.print_given_posts_page())