def test_conversion(self):
md = 'Hello _[World](http://example.com/)!_'
rst = 'Hello *`World <http://example.com/>`_!*'
html = '<p>Hello <em><a href="http://example.com/">World</a>!</em></p>'
try:
import html2text
except ImportError:
raise SkipTest
try:
import html2rest
except ImportError:
try:
helpers.system(['which', 'pandoc'])
except:
raise SkipTest
else:
raise SkipTest
assert imprt.convert(html, fmt='Markdown') == (md, 'markdown')
assert imprt.convert(html, fmt='reStructuredText') == (rst, 'rst')
assert imprt.convert(html, fmt='HTML') == (html, 'html')
assert imprt.convert('', fmt='Markdown') == ('', 'markdown')
assert imprt.convert(None, fmt='Markdown') == ('', 'markdown')
def conversion(self):
md = 'Hello _[World](http://example.com/)!_'
rst = 'Hello *`World <http://example.com/>`_!*'
html = '<p>Hello <em><a href="http://example.com/">World</a>!</em></p>'
try:
import html2text
except ImportError:
return
try:
import html2rest
except ImportError:
try:
helpers.system(['which', 'pandoc'])
except:
return
else:
return
assert imprt.convert(html, fmt='Markdown') == (md, 'markdown')
assert imprt.convert(html, fmt='reStructuredText') == (rst, 'rst')
assert imprt.convert(html, fmt='HTML') == (html, 'html')
assert imprt.convert('', fmt='Markdown') == ('', 'markdown')
assert imprt.convert(None, fmt='Markdown') == ('', 'markdown')
def test_system(self):
examples = ((["echo", "ham"], None, "ham"), ("cat", "foo", "foo"))
for cmd, stdin, expected in examples:
assert helpers.system(cmd, stdin) == expected
with self.assertRaises(AcrylamidException):
helpers.system("false")
with self.assertRaises(OSError):
helpers.system("foo", None)
def test_system(self):
examples = ((['echo', 'ham'], None, 'ham'),
('bc', '1 + 1\n', '2'),
)
for cmd, stdin, expected in examples:
self.assertEqual(helpers.system(cmd, stdin), expected)
with self.assertRaises(AcrylamidException):
helpers.system('bc', '1+1')
with self.assertRaises(OSError):
helpers.system('foo', None)
def system(self):
examples = ((['echo', 'ham'], None, 'ham'),
('cat', 'foo', 'foo'),
)
for cmd, stdin, expected in examples:
assert helpers.system(cmd, stdin) == expected
with attest.raises(AcrylamidException):
helpers.system('false')
with attest.raises(OSError):
helpers.system('foo', None)
def test_system(self):
examples = (
(['echo', 'ham'], None, 'ham'),
('cat', 'foo', 'foo'),
)
for cmd, stdin, expected in examples:
assert helpers.system(cmd, stdin) == expected
with self.assertRaises(AcrylamidException):
helpers.system('false')
with self.assertRaises(OSError):
helpers.system('foo', None)
def run(cmd, ns, src, dest=None):
"""Execute `cmd` such as `yui-compressor %1 -o %2` in-place.
If `dest` is none, you don't have to supply %2."""
assert '%1' in cmd
cmd = cmd.replace('%1', src)
if dest:
assert '%2' in cmd
cmd = cmd.replace('%2', dest)
if not isdir(dirname(dest)):
os.makedirs(dirname(dest))
try:
rv = system(cmd, shell=True)
except (AcrylamidException, OSError):
log.exception("uncaught exception during execution")
return
if dest is None:
fd, path = mkstemp()
with io.open(fd, 'w', encoding='utf-8') as fp:
fp.write(rv)
shutil.move(path, src)
log.info('update %s', src)
else:
log.info('create %s', dest)
def execute(cmd, ns, src, dest=None):
"""Execute `cmd` such as `yui-compressor %1 -o %2` in-place.
If `dest` is none, you don't have to supply %2."""
assert '%1' in cmd
cmd = cmd.replace('%1', src)
if dest:
assert '%2' in cmd
cmd = cmd.replace('%2', dest)
if not isdir(dirname(dest)):
os.makedirs(dirname(dest))
try:
rv = system(cmd, shell=True)
except (AcrylamidException, OSError):
log.exception("uncaught exception during execution")
return
if dest is None:
fd, path = mkstemp()
with io.open(fd, 'w', encoding='utf-8') as fp:
fp.write(rv)
shutil.move(path, src)
log.info('update %s', src)
else:
log.info('create %s', dest)
def write(self, src, dest, force=False, dryrun=False):
dest = dest.replace(splitext(src)[-1], self.target)
if not force and isfile(dest) and getmtime(dest) > getmtime(src):
return event.skip(ns, dest)
if isinstance(self.cmd, basestring):
self.cmd = [self.cmd, ]
tt = time.time()
fd, path = mkstemp(dir=core.cache.cache_dir)
# make destination group/world-readable as other files from Acrylamid
os.chmod(path, os.stat(path).st_mode | stat.S_IRGRP | stat.S_IROTH)
try:
res = helpers.system(self.cmd + [src])
except (OSError, AcrylamidException) as e:
if isfile(dest):
os.unlink(dest)
log.exception('%s: %s' % (e.__class__.__name__, e.args[0]))
else:
with os.fdopen(fd, 'w') as fp:
fp.write(res)
with io.open(path, 'rb') as fp:
mkfile(fp, dest, time.time()-tt, ns, force, dryrun)
finally:
os.unlink(path)
def write(self, src, dest, force=False, dryrun=False):
dest = dest.replace(self.ext, self.target)
if not force and isfile(dest) and getmtime(dest) > getmtime(src):
return event.skip(dest)
if isinstance(self.cmd, basestring):
self.cmd = [self.cmd, ]
tt = time.time()
fd, path = mkstemp(dir=core.cache.cache_dir)
try:
res = helpers.system(self.cmd + [src])
except (OSError, AcrylamidException) as e:
if isfile(dest):
os.unlink(dest)
log.warn('%s: %s' % (e.__class__.__name__, e.args[0]))
else:
with os.fdopen(fd, 'w') as fp:
fp.write(res)
with io.open(path, 'rb') as fp:
mkfile(fp, dest, ctime=time.time()-tt, force=force, dryrun=dryrun)
finally:
os.unlink(path)
def transform(self, text, entry, *args):
try:
system(['which', 'pandoc'])
except AcrylamidException:
if self.ignore:
return text
raise AcrylamidException('Pandoc: pandoc not available')
if len(args) == 0:
raise AcrylamidException("pandoc filter takes one or more arguments")
fmt, extras = args[0], args[1:]
cmd = ['pandoc', '-f', fmt, '-t', 'HTML']
cmd.extend(['--'+x for x in extras])
try:
return system(cmd, stdin=text)
except OSError as e:
raise AcrylamidException(e.msg)
def transform(self, text, entry, *args):
try:
system(['which', 'pandoc'])
except AcrylamidException:
if self.ignore:
return text
raise AcrylamidException('Pandoc: pandoc not available')
if len(args) == 0:
raise AcrylamidException("pandoc filter takes one or more arguments")
fmt, extras = args[0], args[1:]
cmd = ['pandoc', '-f', fmt, '-t', 'HTML']
cmd.extend(['--'+x for x in extras])
try:
return system(cmd, stdin=text)
except OSError as e:
raise AcrylamidException(e.msg)
def convert(data, fmt='markdown', pandoc=False):
"""Reconversion of HTML to Markdown or reStructuredText. Defaults to Markdown,
but can be in fact every format pandoc supports. If pandoc is not available, try
some specific conversion tools like html2text and html2rest.
:param html: raw content to convert to
:param html: format to reconvert to"""
if fmt in ('Markdown', 'markdown', 'mkdown', 'md', 'mkd'):
cmds = ['html2text']
fmt = 'markdown'
elif fmt in ('rst', 'restructuredtext', 'rest', 'reStructuredText'):
cmds = ['html2rest']
fmt = 'rst'
else:
cmds = []
p = [
'pandoc', '--normalize', '-f', 'html', '-t', fmt, '--strict',
'--no-wrap', '--parse-raw'
]
cmds.insert(0, p) if pandoc or fmt == 'rst' else cmds.append(p)
if fmt == 'html':
return data, 'html'
# - item.find(foo).text returns None if no CDATA
# - pandoc waits for input if a zero-length string is given
if data is None or data is '':
return '', fmt
for cmd in cmds:
try:
return system(cmd, stdin=data), fmt.lower()
except AcrylamidException as e:
log.warn(e.args[0])
except OSError:
pass
else:
return data, 'html'
def convert(data, fmt='markdown', pandoc=False):
"""Reconversion of HTML to Markdown or reStructuredText. Defaults to Markdown,
but can be in fact every format pandoc supports. If pandoc is not available, try
some specific conversion tools like html2text and html2rest.
:param html: raw content to convert to
:param html: format to reconvert to"""
if fmt in ('Markdown', 'markdown', 'mkdown', 'md', 'mkd'):
cmds = ['html2text']
fmt = 'markdown'
elif fmt in ('rst', 'restructuredtext', 'rest', 'reStructuredText'):
cmds = ['html2rest']
fmt = 'rst'
else:
cmds = []
p = ['pandoc', '--normalize', '-f', 'html', '-t', fmt, '--strict', '--no-wrap', '--parse-raw']
cmds.insert(0, p) if pandoc or fmt == 'rst' else cmds.append(p)
if fmt == 'html':
return data, 'html'
# - item.find(foo).text returns None if no CDATA
# - pandoc waits for input if a zero-length string is given
if data is None or data is '':
return '', fmt
for cmd in cmds:
try:
return system(cmd, stdin=data), fmt.lower()
except AcrylamidException as e:
log.warn(e.args[0])
except OSError:
pass
else:
return data, 'html'
def convert(data, fmt="markdown", pandoc=False):
"""Reconversion of HTML to Markdown or reStructuredText. Defaults to Markdown,
but can be in fact every format pandoc supports. If pandoc is not available, try
some specific conversion tools like html2text and html2rest.
:param html: raw content to convert to
:param html: format to reconvert to"""
if fmt in ("Markdown", "markdown", "mkdown", "md", "mkd"):
cmds = ["html2text"]
fmt = "markdown"
elif fmt in ("rst", "restructuredtext", "rest", "reStructuredText"):
# cmds = ['html2rest']
fmt = "rst"
else:
cmds = []
p = ["pandoc", "--normalize", "-f", "html", "-t", fmt, "--strict", "--no-wrap"]
cmds.insert(0, p) if pandoc else cmds.append(p)
if fmt == "html":
return data, "html"
# - item.find(foo).text returns None if no CDATA
# - pandoc waits for input if a zero-length string is given
if data is None or data is "":
return "", fmt
for cmd in cmds:
try:
return system(cmd, stdin=data), fmt.lower()
except AcrylamidException as e:
log.warn(e.args[0])
except OSError:
pass
else:
return data, "html"
def write(self, src, dest, force=False, dryrun=False):
dest = dest.replace(self.ext, self.target)
if not force and isfile(dest) and getmtime(dest) > getmtime(src):
return event.skip(dest)
if isinstance(self.cmd, basestring):
self.cmd = [self.cmd, ]
tt = time.time()
fd, path = mkstemp(dir=core.cache.cache_dir)
try:
res = helpers.system(self.cmd + [src])
except OSError:
if isfile(dest):
os.unlink(dest)
raise AcrylamidException('%s is not available!' % self.cmd[0])
with os.fdopen(fd, 'w') as fp:
fp.write(res)
with io.open(path, 'rb') as fp:
mkfile(fp, dest, ctime=time.time()-tt, force=force, dryrun=dryrun, mode="b")
