def apply_creole(content, page_msg):
"""
Use python-creole:
http://code.google.com/p/python-creole/
We used verbose for insert error information (e.g. not existing macro)
into the generated page
"""
from creole import creole2html
from pylucid_project.apps.pylucid.markup import PyLucid_creole_macros
if settings.DEBUG:
verbose = 2
else:
verbose = 1
try:
# new python-creole v1.0 API
return creole2html(content, macros2=PyLucid_creole_macros, stderr=page_msg, verbose=verbose)
except TypeError:
# TODO: Remove work-a-round in future release
# old python-creole API
emitter_kwargs = {
"macros":PyLucid_creole_macros,
"verbose": verbose,
"stderr": page_msg
}
return creole2html(content, emitter_kwargs=emitter_kwargs)
def test_stderr(self):
"""
Test if the traceback information send to a stderr handler.
"""
my_stderr = StringIO()
creole2html(
markup_string="<<notexist1>><<notexist2>><</notexist2>>",
emitter_kwargs={
"verbose":2,
"stderr":my_stderr,
}
)
error_msg = my_stderr.getvalue()
# Note:
# The error message change if macros are a dict or are a object!
# Check if we get a traceback information into our stderr handler
must_have = (
"Traceback", "'notexist1'", "'notexist2'",
)
for part in must_have:
tb_lines = [" -"*40]
tb_lines += error_msg.splitlines()
tb_lines += [" -"*40]
tb = "\n".join([" >>> %s" % l for l in tb_lines])
msg = "%r not found in:\n%s" % (part, tb)
# TODO: use assertIn if python 2.6 will be not support anymore.
if part not in error_msg:
raise self.failureException(msg)
def test_lineendings(self):
""" Test all existing lineending version """
out_string = creole2html("first\nsecond")
self.assertEqual(out_string, "<p>first<br />\nsecond</p>")
out_string = creole2html("first\rsecond")
self.assertEqual(out_string, "<p>first<br />\nsecond</p>")
out_string = creole2html("first\r\nsecond")
self.assertEqual(out_string, "<p>first<br />\nsecond</p>")
def parse(self, to_parse):
content = ''
value = getattr(self, to_parse)
if self.parser == 'html':
content = value
elif self.parser == 'markdown':
try:
import markdown
content = markdown.markdown(value, extensions=['markdown.extensions.codehilite',
'markdown.extensions.fenced_code',
'markdown.extensions.tables'])
except ImportError:
content = value
print('Markdown parser not installed. Try `pip install Markdown`')
elif self.parser == 'textile':
try:
import textile
content = textile.textile(value)
except ImportError:
content = value
print('Markdown parser not installed. Try `pip install textile`')
elif self.parser == 'mediawiki':
try:
from creole import creole2html
content = creole2html(unicode(value))
except ImportError:
content = value
print('Markdown parser not installed. Try `pip install python-creole`')
return content
def parse(self, value, context=None, placeholder=None):
try:
import creole
except ImportError:
return value
else:
return creole.creole2html(value)
def test_default_macro3(self):
html = creole2html(
markup_string=u"<<html>>1<</html>><<html>>2<</html>>",
verbose=1,
# stderr=sys.stderr, debug=False
)
self.assertEqual(html, u"1\n2\n")
def test_wiki_style_line_breaks3(self):
html = creole2html(
markup_string=self._prepare_text("""
with blog line breaks, every line break would be convertet into<br />
with wiki style not.
This is the first line,\\\\and this is the second.
new line
block 1
new line
block 2
end
"""),
parser_kwargs={"blog_line_breaks":False},
)
self.assertEqual(html, self._prepare_text("""
<p>with blog line breaks, every line break would be convertet into<br /> with wiki style not.</p>
<p>This is the first line,<br />
and this is the second.</p>
<p>new line block 1</p>
<p>new line block 2</p>
<p>end</p>
"""))
def assert_Creole2html(self, source_string, should_string, \
verbose=1, stderr=sys.stderr, debug=False):
"""
compare the generated html code from the markup string >source_string<
with the >should_string< reference.
"""
self.assertNotEqual(source_string, should_string)
# prepare whitespace on test strings
markup_string = self._prepare_text(source_string)
assert isinstance(markup_string, unicode)
should = self._prepare_text(should_string)
assert isinstance(should, unicode)
if debug:
self._debug_text("assert_Creole2html() should_string", should)
# convert creole markup into html code
out_string = creole2html(
markup_string, verbose=verbose, stderr=stderr, debug=debug
)
if debug:
self._debug_text("assert_Creole2html() creole2html", out_string)
out_string = out_string.rstrip("\n")
out_string = out_string.replace("\t", " ")
# compare
self.assertEqual(out_string, should)
def test_default_macro2(self):
html = creole2html(
markup_string=u"<<html>>{{{<nocode>}}}<</html>>",
verbose=1,
# stderr=sys.stderr, debug=False
)
self.assertEqual(html, u"{{{<nocode>}}}\n")
def read(self, source_path):
"""Parse content and metadata of creole files"""
self._metadata = {}
with pelican_open(source_path) as text:
content = creole2html(text, macros={'header': self._parse_header_macro})
return content, self._metadata
def test_macro_wrong_arguments_with_error_report(self):
"""
simple test for the "macro API"
"""
def test(text, foo):
pass
my_stderr = StringIO()
html = creole2html(
markup_string="<<test bar='foo'>>c<</test>>",
emitter_kwargs={
"verbose":2,
"macros":{"test":test},
"stderr":my_stderr,
}
)
self.assertEqual(html,
"[Error: Macro 'test' error: test() got an unexpected keyword argument 'bar']"
)
error_msg = my_stderr.getvalue()
# Check traceback information into our stderr handler
must_have = (
"TypeError: test() got an unexpected keyword argument 'bar'",
"sourceline: 'def test(text, foo):' from",
"tests/test_creole2html.py",
)
for part in must_have:
self.assertIn(part, error_msg)
def wiki(page):
store = get_db()
if not page in store:
flask.abort(404)
else:
text = store[page].decode()
morphed_text = creole.creole2html(text)
return flask.render_template("entry.html", key=page, value=morphed_text)
def get_long_description(package_root, filename="README.creole", raise_errors=None):
""" read README file and convert it on-the-fly to ReStructuredText. """
if raise_errors is None:
raise_errors = should_raise_errors()
if raise_errors:
sys.stderr.write("Test creole2rest and raise an error, if rendering failed...\n")
# raise a error if a unknown node found
unknown_emit = raise_unknown_node
else:
# ignore unknown nodes
unknown_emit = transparent_unknown_nodes
filepath = os.path.join(package_root, filename)
long_description_origin = ""
try:
# Read creole README
f = codecs.open(filepath, "r", encoding="utf-8")
long_description_origin = f.read().strip()
f.close()
# convert creole into html
long_description_html = creole2html(long_description_origin)
# convert html to ReSt
long_description_rest_unicode = html2rest(
long_description_html,
emitter_kwargs={"unknown_emit":unknown_emit}
)
if PY3:
long_description_rest = long_description_rest_unicode
else:
long_description_rest = long_description_rest_unicode.encode("utf-8")
except Exception:
if raise_errors:
raise
# Don't raise the error e.g. in ./setup install process
evalue = sys.exc_info()[1]
long_description_rest = "[Error: %s]\n%s" % (
evalue, long_description_origin
)
else:
if raise_errors:
# Test created ReSt code like PyPi does it.
from creole.rest_tools.pypi_rest2html import pypi_rest2html
try:
pypi_rest2html(long_description_rest_unicode)
except SystemExit as e:
msg = "Error creole2rest self test failed: rest2html() exist with status code: %s\n" % e.args[0]
sys.stderr.write(msg)
sys.exit(msg)
except Exception as e:
sys.exit("ReSt2html error: %s" % e)
else:
if "check" in sys.argv:
print("Generating creole to ReSt to html, ok.")
return long_description_rest
def test_headline_spaces(self):
"""
https://code.google.com/p/python-creole/issues/detail?id=15
"""
html = creole2html(markup_string="== Headline1 == \n== Headline2== ")
self.assertEqual(html, self._prepare_text("""
<h2>Headline1</h2>
<h2>Headline2</h2>
"""))
def test_stderr(self):
"""
Test if the traceback information send to a stderr handler.
"""
my_stderr = StringIO.StringIO()
creole2html(markup_string=u"<<notexist1>><<notexist2>><</notexist2>>", verbose=2, stderr=my_stderr, debug=False)
error_msg = my_stderr.getvalue()
# Check if we get a traceback information into our stderr handler
must_have = (
"<pre>",
"</pre>",
"Traceback",
"AttributeError:",
"has no attribute 'notexist1'",
"has no attribute 'notexist2'",
)
for part in must_have:
self.failUnless(part in error_msg, "String %r not found in:\n******\n%s******" % (part, error_msg))
def test_example_macros2(self):
html = creole2html(
markup_string="<<html>>{{{<nocode>}}}<</html>>",
emitter_kwargs={
"verbose":1,
"macros":example_macros,
"stderr":sys.stderr,
}
)
self.assertEqual(html, '{{{<nocode>}}}')
def test_default_macro1(self):
"""
Test the default "html" macro, found in ./creole/default_macros.py
"""
html = creole2html(
markup_string=u"<<html>><p>foo</p><</html>><bar?>",
verbose=1,
# stderr=sys.stderr, debug=False
)
self.assertEqual(html, u"<p>foo</p>\n<p><bar?></p>\n")
def test_macro_dict(self):
"""
simple test for the "macro API"
"""
def test(args, text):
return u"XXX%s|%sXXX" % (args, text)
html = creole2html(markup_string=u"<<test foo=1>>bar<</test>>", macros={"test": test})
self.assertEqual(html, u"XXXfoo=1|barXXX\n")
def test_example_macros3(self):
html = creole2html(
markup_string="<<html>>1<</html>><<html>>2<</html>>",
emitter_kwargs={
"verbose":1,
"macros":example_macros,
"stderr":sys.stderr,
}
)
self.assertEqual(html, '1\n2')
def creole(value):
"Converts Creole-formatted text to formatted HTML."
try:
import creole
except ImportError:
if settings.DEBUG:
raise template.TemplateSyntaxError("Error in {% creole %} filter: The Python creole library isn't installed.")
return force_unicode(value)
else:
return mark_safe(force_unicode(creole.creole2html(value)))
def render(self, name, value, attrs=None):
if value is None:
value = ''
final_attrs = self.build_attrs(attrs, name=name)
return format_html(
"""<textarea{0}>\r\n{1}</textarea>
<div class="restructed-preview">%s</div>
""" % creole.creole2html(unicode(value)),
flatatt(final_attrs),
force_text(value)
)
def test_macro_class(self):
"""
simple test for the "macro API"
"""
class TestMacro(object):
def test(self, args, text):
return u"XXX%s|%sXXX" % (args, text)
html = creole2html(markup_string=u"<<test foo=1>>bar<</test>>", macros=TestMacro())
self.assertEqual(html, u"XXXfoo=1|barXXX\n")
def highlight(body, syntax):
"""
Have Pygments format the given text.
"""
if syntax == 'creole':
return '<div class="creole">%s</div>' % (
creole.creole2html(body, blog_line_breaks=False),
)
if syntax == 'csv':
return csv_to_html(body)
lexer = pygments.lexers.get_lexer_by_name(syntax)
return pygments.highlight(body, lexer, FORMATTER)
def test_macro_callable(self):
"""
simple test for the "macro API"
"""
def testmacro(macroname, args, text):
if macroname == "test":
return u"XXX%s|%sXXX" % (args, text)
raise AssertionError("Wrong macro name?")
html = creole2html(markup_string=u"<<test foo=1>>bar<</test>>", macros=testmacro)
self.assertEqual(html, u"XXXfoo=1|barXXX\n")
def test_example_macros1(self):
"""
Test the default "html" macro, found in ./creole/default_macros.py
"""
html = creole2html(
markup_string="<<html>><p>foo</p><</html>><bar?>",
emitter_kwargs={
"verbose":1,
"macros":example_macros,
"stderr":sys.stderr,
}
)
self.assertEqual(html, '<p>foo</p>\n<p><bar?></p>')
def get_long_description(package_root, filename="README.creole", raise_errors=None):
""" read README file and convert it on-the-fly to ReStructuredText. """
if raise_errors is None:
raise_errors = should_raise_errors()
if raise_errors:
# raise a error if a unknown node found
unknown_emit = raise_unknown_node
else:
# ignore unknown nodes
unknown_emit = transparent_unknown_nodes
filepath = os.path.join(package_root, filename)
long_description_origin = ""
try:
# Read creole README
f = codecs.open(filepath, "r", encoding="utf-8")
long_description_origin = f.read().strip()
f.close()
# convert creole into html
long_description_html = creole2html(long_description_origin)
# convert html to ReSt
long_description_rest_unicode = html2rest(
long_description_html,
emitter_kwargs={"unknown_emit":unknown_emit}
)
if PY3:
long_description_rest = long_description_rest_unicode
else:
long_description_rest = long_description_rest_unicode.encode("utf-8")
except Exception:
if raise_errors:
raise
# Don't raise the error e.g. in ./setup install process
evalue = sys.exc_info()[1]
long_description_rest = "[Error: %s]\n%s" % (
evalue, long_description_origin
)
else:
if raise_errors:
# Test created ReSt code
from creole.rest2html.clean_writer import rest2html
try:
rest2html(long_description_rest_unicode, traceback=True, enable_exit_status=1, exit_status_level=2)
except SystemExit as e:
print("Error creole2rest self test failed: rest2html() exist with status code: %s" % e.args[0])
raise
return long_description_rest
def run(self):
tz = pytz.timezone("Asia/Taipei")
#list_urls = self.getListUrls() #拼接明星列表页url
#actor_ids = self.getActorIds(list_urls) #获取艺人ID号
#ids = ','.join(actor_ids)
#f = open('sina_actor_ids.txt', 'w+')
#f.write(ids)
#f.close
#actor_ids = ['7915']
f = open('sina_actor_ids.txt', 'r')
actor_content = f.read()
actor_ids = actor_content.split(',')
f.close
for actor_id in actor_ids:
actor_data = ''
actor_data = self.getActorBaseInfo(actor_id)
if actor_data:
actor_data['model'] = 'actor'
actor_data['created_at'] = datetime.now().replace(tzinfo=tz)
actor_data['updated_at'] = datetime.now().replace(tzinfo=tz)
actor_data['title'] = actor_data['title'].replace('-', '·')
print actor_data
else:
print '跳过组合'
continue
actor_content = self.getActorContent(actor_id)
print actor_content
actor_data['content'] = html2creole(actor_content)
#过滤掉自动生成的wiki链接
actor_data['content'] = actor_data['content'].replace('[[', '')
actor_data['content'] = actor_data['content'].replace(']]', '')
actor_data['html_cache'] = creole2html(actor_data['content'])
slug = proposal = re.sub("[^\x61-\xff\d\w]", "-", actor_data['title'].decode("utf-8").encode("gb18030", "ignore")).decode("gb18030", "ignore").encode("utf-8").strip("-")
similarSlugs = []
regex = re.compile("^%s" % slug)
for result in self.mongo_wiki.find({"slug": regex}):
similarSlugs.append(result['slug'])
i = 1
while slug in similarSlugs:
i = i + 1
slug = proposal + "-" + str(i)
actor_data['slug'] = slug
self.mongo_wiki.insert(actor_data)
def render(readme_info):
"""
Turns a readme file into HTML
:param readme_info:
The dict from ReadmeClient, containing the keys:
`format` - one of `markdown`, `textile`, `creole`, `rst`, `txt`
`contents` - unicode/str contents of readme
:return:
An HTML string
"""
contents = re.sub("\r\n", "\n", readme_info["contents"])
if readme_info["format"] == "markdown":
contents = contents.replace("\t", " ")
output = _markdown(contents)
elif readme_info["format"] == "textile":
output = textile.textile(contents, html_type="html")
elif readme_info["format"] == "creole":
output = creole2html(contents)
elif readme_info["format"] == "rst":
output = rest2html(contents, report_level=4)
# Everything else is treated as txt
else:
output = contents
for char, entity in _entities:
output = output.replace(char, entity)
output = output.replace("\n", "<br>\n")
output = sanitize(output)
if output.find("src=") != -1 or output.find("href=") != -1:
url_dirname = os.path.dirname(readme_info["url"]) + "/"
src_dirname = url_dirname
href_dirname = url_dirname
github_match = re.match("https://raw.githubusercontent.com(/[^/]+/[^/]+/)(.*$)", url_dirname)
if github_match is not None:
href_dirname = "https://github.com" + github_match.group(1) + "blob/" + github_match.group(2)
output = re.sub("(<img\\s+[^>]*\\bsrc=[\"'])(?!http://|https://|/)", "\\1" + src_dirname, output, 0, re.I)
output = re.sub(
"(<a\\s+[^>]*\\bhref=[\"'])(?!http://|https://|/|mailto:|#)", "\\1" + href_dirname, output, 0, re.I
)
return output
def test_wiki_style_line_breaks1(self):
html = creole2html(
markup_string=self._prepare_text("""
wiki style
linebreaks
...and not blog styled.
"""),
parser_kwargs={"blog_line_breaks":False},
)
self.assertEqual(html, self._prepare_text("""
<p>wiki style linebreaks</p>
<p>...and not blog styled.</p>
"""))
def test_stderr(self):
"""
Test if the traceback information send to a stderr handler.
"""
my_stderr = StringIO.StringIO()
creole2html(
markup_string=u"<<notexist1>><<notexist2>><</notexist2>>",
emitter_kwargs={
"verbose":2,
"stderr":my_stderr,
}
)
error_msg = my_stderr.getvalue()
# Check if we get a traceback information into our stderr handler
must_have = (
"Traceback",
"AttributeError:",
"has no attribute 'notexist1'",
"has no attribute 'notexist2'",
)
for part in must_have:
self.assertIn(part, error_msg)
def render_text(self):
return mark_safe(creole2html(self.text))
def render_content(self):
return mark_safe(creole2html(self.content))
def creole2html(creole_text):
html = cr.creole2html(unicode(creole_text), macros=macros)
return html
def render(self, text, **kwargs):
from creole import creole2html
return creole2html(text)
def html(self, input_text):
import creole
return creole.creole2html(input_text)
def remove_creole(article):
title, text = article
text = creole2html(text)
return title, text
def index(request, lemma):
#
# Check if we found something in our own sparql repository. If not
# query other sources.
#
# TODO: We need a better check (persons with the same name).
#
#if not sparql_results or not sparql_results["results"]["bindings"]:
if False:
#
# DBPEDIA
#
sparql = SPARQLWrapper(DBPEDIA_QUERY_URL)
sparql.setQuery(SPARQL_DBPEDIA_QUERY.format(lemma))
sparql.setReturnFormat(JSON)
try:
sparql_results = sparql.queryAndConvert()
except:
import traceback
print traceback.format_exc()
sparql_results = {}
#if sparql_results and sparql_results["results"]["bindings"]:
# for result in sparql_results["results"]["bindings"]:
# from .utils import sparql_local_insert_person
#
# sparql_local_insert_person(lemma, result)
#else:
if True:
#
# CBDB
#
r = requests.get(CBDB_API_URL.format(lemma)).json()
#if r.status_code == 200:
try:
persons = r['Package']['PersonAuthority']['PersonInfo'][
'Person']
except:
persons = []
if type(persons) is list:
for person in persons:
print person['BasicInfo']['ChName'], person['BasicInfo'][
'YearBirth'], person['BasicInfo']['PersonId']
else:
person = persons
if person:
print person['BasicInfo']['ChName'], person['BasicInfo'][
'YearBirth'], person['BasicInfo']['PersonId']
sparql = SPARQLWrapper(FUSEKI_QUERY_URL)
sparql.setQuery(sparql_query.format(lemma))
sparql.setReturnFormat(JSON)
try:
sparql_results = sparql.queryAndConvert()
except:
sparql_results = {}
sparql = SPARQLWrapper(FUSEKI_QUERY_URL)
sparql.setQuery(sparql_query.format(lemma))
sparql.setReturnFormat(JSON)
try:
sparql_results = sparql.queryAndConvert()
except:
sparql_results = {}
is_person = False
template_result = {}
if sparql_results.get("results", False):
for result in sparql_results["results"]["bindings"]:
p = result["p"]["value"].replace(prefix_default, '')
p = p.replace(prefix_schema, '')
p = p.replace(prefix_syntax, '')
o = result["o"]["value"].replace(prefix_default, '')
if p == "type" and o == "Person":
is_person = True
template_result[p] = o
template_result['is_person'] = is_person
template_result['lemma'] = lemma
# Wikipedia
try:
wikipedia.set_lang("en")
en = wikipedia.page(lemma, auto_suggest=True, redirect=True)
wikipedia_en = en.summary
wikipedia_en_url = en.url
except:
wikipedia_en = ''
wikipedia_en_url = ''
try:
wikipedia.set_lang("zh")
zh = wikipedia.page(lemma, auto_suggest=True, redirect=True)
wikipedia_zh = zh.summary
wikipedia_zh_url = zh.url
except:
wikipedia_zh = ''
wikipedia_zh_url = ''
# Sinology
try:
f = codecs.open("/docker/dublin-store/sinology/mainSpace/" + lemma,
"r", "utf-8")
# Skip first line
sinology = f.readlines()[1:]
sinology = '\n'.join(sinology)
sinology = creole.creole2html(sinology)
except:
sinology = ''
return render(
request, 'sparql/index.html', {
'r': template_result,
'wikipedia_en': wikipedia_en,
'wikipedia_zh': wikipedia_zh,
'wikipedia_en_url': wikipedia_en_url,
'wikipedia_zh_url': wikipedia_zh_url,
'sinology': sinology,
})
def _parse_text(self):
self.text = creole2html(self.text)
* **creole2html**, **html2creole**, **html2rest**, //html2textile//
=== a table:
|=headline 1 |= headline 2 |
| 1.1. cell | 1.2. cell |
| 2.1. cell | 2.2. cell |
----
More info on our [[http://code.google.com/p/python-creole/|Homepage]]."""
if __name__ == "__main__":
print("_" * 79 + "\n*** Convert creole into html: ***\n\n")
html = creole2html(source_creole)
print(html)
print("\n\n" + "_" * 79 + "\n*** Convert html back into creole: ***\n\n")
creole = html2creole(html)
print(creole)
print("\n\n" + "_" * 79 +
"\n*** Convert html into ReStructuredText: ***\n\n")
rest = html2rest(html)
print(rest)
print("\n\n" + "_" * 79 + "\n*** Convert html into textile: ***\n\n")
textile = html2textile(html)
print(textile)
def render_teaser(self):
return mark_safe(creole2html(self.teaser))
def test_creole_basic(self):
out_string = creole2html("a text line.")
self.assertEqual(out_string, "<p>a text line.</p>")
def to_html(self, text):
html = creole2html(text, macros=self._macros)
return self._extract_meta(html)