示例#1
0
def html2rst(text, images_dir):
    """Converts html, tipically generated by tinyMCE, into rst
    compatible with Sage documentation.

    The main job is done by BeautifulSoup, which is much more
    robust than conventional parsers like HTMLParser, but also
    several details specific of this context are taken into
    account, so this code differs from generic approaches like
    those found on the web.

    INPUT:

    - ``text`` -- string -- a chunk of HTML text

    - ``images_dir`` -- string -- folder where images are stored

    OUTPUT:

    - string -- rst text

    EXAMPLES::

        sage: from sagenb.misc.comments2rst import html2rst # optional - beautifulsoup
        sage: html2rst('<p>Some text with <em>math</em>: $e^{\pi i}=-1$</p>', '') # optional - beautifulsoup
        u'Some text with  *math* :  :math:`e^{\\pi i}=-1`\n\n'
        sage: html2rst('<p>Text with <em>incorrect</p> nesting</em>.', '') # optional - beautifulsoup
        u'Text with  *incorrect* \n\n nesting\n.'
        sage: html2rst('<pre>Preformatted: \n    a+2\n</pre><p> Not preformatted: \n    a+2\n</p>', '') # optional - beautifulsoup
        u'::\n\n    Preformatted: \n        a+2\n    \n Not preformatted:     a\\+2\n\n'
        sage: html2rst('&aacute;ñ&nbsp;&ntildeá','') # optional - beautifulsoup
        u'\xe1\xf1 \xf1\xe1'
        sage: html2rst('<p>some text</p><p>$$</p><p>3.183098861 \cdot 10^{-1}</p><p>$$</p>','') # optional - beautifulsoup
        u'some text\n\n.. MATH::\n\n    3.183098861 \\cdot 10^{-1}\n\n.. end of math\n\n'
    """
    
    #replace $$some display latex$$ with
    #<display>some display latex</display>
    text = preprocess_display_latex(text)

    #eliminate nasty &nbsp;
    text = text.replace('&nbsp;',' ')
            
    #ICantBelieveItsBeautifulSoup is better than BeautifulSoup
    #for html that wasn't generated by humans (like tinyMCE)
    soup = ICantBelieveItsBeautifulSoup(text,
                       convertEntities=ICantBelieveItsBeautifulSoup.ALL_ENTITIES)        

    #remove all comments
    comments = soup.findAll(text=lambda text:isinstance(text, Comment))
    for comment in comments:
        comment.extract()

#    replace_courier(soup)
    replace_latex(soup)
    v = Soup2Rst(images_dir)

#    return v.visit(soup)
    text = v.visit(soup)
    more_than_2_blank_lines = re.compile(r'\n\n+', re.MULTILINE)
    text = more_than_2_blank_lines.sub('\n\n', text)
    text = replace_xml_entities(text)
    return text
示例#2
0
def html2rst(text, images_dir):
    """Converts html, tipically generated by tinyMCE, into rst
    compatible with Sage documentation.

    The main job is done by BeautifulSoup, which is much more
    robust than conventional parsers like HTMLParser, but also
    several details specific of this context are taken into
    account, so this code differs from generic approaches like
    those found on the web.

    INPUT:

    - ``text`` -- string -- a chunk of HTML text

    - ``images_dir`` -- string -- folder where images are stored

    OUTPUT:

    - string -- rst text

    EXAMPLES::

        sage: from sagenb.misc.comments2rst import html2rst # optional - beautifulsoup
        sage: html2rst('<p>Some text with <em>math</em>: $e^{\pi i}=-1$</p>', '') # optional - beautifulsoup
        u'Some text with  *math* :  :math:`e^{\\pi i}=-1`\n\n'
        sage: html2rst('<p>Text with <em>incorrect</p> nesting</em>.', '') # optional - beautifulsoup
        u'Text with  *incorrect* \n\n nesting\n.'
        sage: html2rst('<pre>Preformatted: \n    a+2\n</pre><p> Not preformatted: \n    a+2\n</p>', '') # optional - beautifulsoup
        u'::\n\n    Preformatted: \n        a+2\n    \n Not preformatted:     a\\+2\n\n'
        sage: html2rst('&aacute;ñ&nbsp;&ntildeá','') # optional - beautifulsoup
        u'\xe1\xf1 \xf1\xe1'
        sage: html2rst('<p>some text</p><p>$$</p><p>3.183098861 \cdot 10^{-1}</p><p>$$</p>','') # optional - beautifulsoup
        u'some text\n\n.. MATH::\n\n    3.183098861 \\cdot 10^{-1}\n\n.. end of math\n\n'
    """

    #replace $$some display latex$$ with
    #<display>some display latex</display>
    text = preprocess_display_latex(text)

    #eliminate nasty &nbsp;
    text = text.replace('&nbsp;', ' ')

    #ICantBelieveItsBeautifulSoup is better than BeautifulSoup
    #for html that wasn't generated by humans (like tinyMCE)
    soup = ICantBelieveItsBeautifulSoup(
        text, convertEntities=ICantBelieveItsBeautifulSoup.ALL_ENTITIES)

    #remove all comments
    comments = soup.findAll(text=lambda text: isinstance(text, Comment))
    for comment in comments:
        comment.extract()

#    replace_courier(soup)
    replace_latex(soup)
    v = Soup2Rst(images_dir)

    #    return v.visit(soup)
    text = v.visit(soup)
    more_than_2_blank_lines = re.compile(r'\n\n+', re.MULTILINE)
    text = more_than_2_blank_lines.sub('\n\n', text)
    text = replace_xml_entities(text)
    return text
示例#3
0
def typygmentdown(text, **kwargs):
    """
    Given a string of text using Markdown syntax, applies the
    following transformations:
    
    1. Searches out and temporarily removes any raw ``<code>``
       elements in the text.
    2. Applies Markdown and typogrify to the remaining text.
    3. Applies Pygments highlighting to the contents of the removed
       ``<code>`` elements.
    4. Re-inserts the ``<code>`` elements and returns the result.
    
    The Pygments lexer to be used for highlighting is determined by
    the ``class`` attribute of each ``<code>`` element found; if none
    is present, it will attempt to guess the correct lexer before
    falling back on plain text.
    
    The following keyword arguments are understood and passed to
    markdown if found:
    
    * ``extensions``
    
    Markdown's ``safe_mode`` argument is *not* passed on, because it
    would cause the temporary ``<code>`` elements in the text to be
    escaped.
    
    The following keyword arguments are understood and passed to
    Pygments if found:
    
    * ``linenos``
    
    The removal, separate highlighting and re-insertion of the
    ``<code>`` elements is necessary because Markdown and SmartyPants
    do not reliably avoid formatting text inside these elements;
    removing them before applying Markdown and typogrify means they
    are in no danger of having extraneous HTML or fancy punctuation
    inserted by mistake.
    
    Original implementation by user 'blinks' as snippet #119 at
    djangosnippets: http://www.djangosnippets.org/snippets/119/. This
    version makes the following changes:

    * The name of the function is now ``typygmentdown``.
    * The argument signature has changed to work better with the
      ``template_utils`` formatter.
    * The ``extensions`` and ``linenos`` arguments are looked for and
      passed to Markdown and Pygments, respectively.
    * The function is registered with the ``template_utils``
      formatter.
    
    """

    #soup = BeautifulSoup(unicode(text))
    #code_blocks = soup.findAll(u'code')
    #for block in code_blocks:
    #    block.replaceWith(u'<code class="removed"></code>')

    text2 = unicode(text)
    code_blocks = []
    converted = []
    start = 0
    end = len(text2)

    # loop over every tag
    reg = re.compile(u'<code.*?code>', re.X  | re.DOTALL)
    for m in reg.finditer(text2):

        # ignore comments
		#if not re.match(r'^!--(.*)--',tag):
        block = m.group(0)
        start_block = m.start()
        end_block = m.end()

        code_blocks.append(block)
        
        if start < start_block:
            converted.append(text2[start:(start_block-1)])

        converted.append(u'<code class="removed"></code>')
        start = end_block + 1

    if start < end:
	    converted.append(text2[start:end])


    htmlized = typogrify(markdown(u''.join(converted), extensions=kwargs.get('extensions', [])))



    ##typogrify()
    soup = ICantBelieveItsBeautifulSoup(htmlized)
    empty_code_blocks, index = soup.findAll('code', 'removed'), 0
    formatter = HtmlFormatter(cssclass='typygmentdown', linenos=kwargs.get('linenos', False))


    for block in code_blocks:
        
        #import pdb
        #pdb.set_trace()

        m = re.search(u"class=\"(?P<class_name>\w+)\">(?P<block_content>.*)</code>", block, re.DOTALL)
       
        block_content = u''
        if m:
            block_group = m.groupdict()
            block_content =  block_group['block_content']
            language = block_group['class_name']
        else:
            language = 'text'
        try:
            lexer = get_lexer_by_name(language, stripnl=True, encoding='UTF-8')
        except ValueError, e:
            try:
                lexer = guess_lexer(block.renderContents())
            except ValueError, e:
                lexer = get_lexer_by_name('text', stripnl=True, encoding='UTF-8')