def test_settings_return_independent(self):
# Make sure that the results from one settings call doesn't
# effect past or future instances.
self.PATH = abspath(dirname(__file__))
default_conf = join(self.PATH, 'default_conf.py')
settings = read_settings(default_conf)
settings['SITEURL'] = 'new-value'
new_settings = read_settings(default_conf)
self.assertNotEqual(new_settings['SITEURL'], settings['SITEURL'])
def setUp(self, settings_overrides=None, count=5,
categories_per_content=1, categories=None):
self.temp_input_dir = tempfile.mkdtemp(prefix="cc-input-")
page_directory = os.path.join(self.temp_input_dir, 'pages')
os.mkdir(page_directory)
self.temp_output_dir = tempfile.mkdtemp(prefix="cc-output-")
if categories is None:
categories = [get_random_text_and_whitespace() for _ in range(5)]
self.articles = make_content(
self.temp_input_dir, categories, count=count,
categories_per_content=categories_per_content)
self.pages = make_content(
page_directory, categories, count=count,
categories_per_content=categories_per_content)
settings = {
'PATH': self.temp_input_dir,
'PAGE_DIR': 'pages',
'OUTPUT_PATH': self.temp_output_dir,
'PLUGINS': [cc],
'DEFAULT_DATE': (2014, 6, 8),
}
if settings_overrides is not None:
settings.update(settings_overrides)
settings = read_settings(override=settings)
pelican = Pelican(settings=settings)
pelican.modified_run = modified_pelican_run
self.collations = pelican.modified_run(pelican)['collations']
def test_turkish_locale(self):
if platform == 'win32':
locale_string = 'Turkish'
else:
locale_string = 'tr_TR.UTF-8'
settings = read_settings(
override={
'LOCALE': locale_string,
'TEMPLATE_PAGES': {
'template/source.html': 'generated/file.html'
}
})
generator = TemplatePagesGenerator(
{'date': self.date}, settings,
self.temp_content, '', self.temp_output)
generator.env.filters.update({'strftime': utils.DateFormatter()})
writer = Writer(self.temp_output, settings=settings)
generator.generate_output(writer)
output_path = os.path.join(
self.temp_output, 'generated', 'file.html')
# output file has been generated
self.assertTrue(os.path.exists(output_path))
# output content is correct
with utils.pelican_open(output_path) as output_file:
self.assertEqual(output_file,
utils.strftime(self.date, 'date = %A, %d %B %Y'))
def test_theme_static_paths_copy(self):
# the same thing with a specified set of settings should work
settings = read_settings(
path=SAMPLE_CONFIG,
override={
"PATH": INPUT_PATH,
"OUTPUT_PATH": self.temp_path,
"CACHE_PATH": self.temp_cache,
"THEME_STATIC_PATHS": [
os.path.join(SAMPLES_PATH, "very"),
os.path.join(SAMPLES_PATH, "kinda"),
os.path.join(SAMPLES_PATH, "theme_standard"),
],
},
)
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
theme_output = os.path.join(self.temp_path, "theme")
extra_path = os.path.join(theme_output, "exciting", "new", "files")
for file in ["a_stylesheet", "a_template"]:
self.assertTrue(os.path.exists(os.path.join(theme_output, file)))
for file in ["wow!", "boom!", "bap!", "zap!"]:
self.assertTrue(os.path.exists(os.path.join(extra_path, file)))
def test_read_empty_settings(self):
# Providing no file should return the default values.
settings = read_settings(None)
expected = copy.deepcopy(_DEFAULT_CONFIG)
expected['FEED_DOMAIN'] = '' # Added by configure settings
self.maxDiff = None
self.assertDictEqual(settings, expected)
def test_custom_generation_works(self):
# the same thing with a specified set of settings should work
pelican = Pelican(path=INPUT_PATH, output_path=self.temp_path,
settings=read_settings(SAMPLE_CONFIG))
pelican.run()
diff = dircmp(self.temp_path, os.sep.join((OUTPUT_PATH, "custom")))
self.assertFilesEqual(diff)
def setUp(self):
self.__sqlite_file = tempfile.NamedTemporaryFile()
self.__store = Store(self.__sqlite_file.name)
self.__pelican_config = os.path.join(os.path.dirname(__file__), "samplesite", "pelicanconf.py")
from pelican.settings import read_settings
settings = read_settings(self.__pelican_config, override={"SITEURL": os.path.abspath(os.curdir)})
root_path = os.path.abspath(os.path.dirname(self.__pelican_config))
self.__store.sync(settings)
from lapis.command import sub_command_classes
parser = ArgumentParser(prog="testlapis", description="TestUtility for performing common pelican tasks.")
# sub-commands
subparsers = parser.add_subparsers()
for command_cls in sub_command_classes:
command_cls.setup(subparsers)
# bogus mock config obj
self.config = type("Config", (object,), {})()
self.config.example_lapis_configuration_file = Config(self.__pelican_config).example_lapis_configuration_file
self.config.settings = settings
self.config.store = self.__store
self.str_io = io.StringIO()
self.config.printer = CommandPrinter(stream=self.str_io)
self.config.editor = TrivialEditor("echo")
self.__tmp_dir = tempfile.mkdtemp()
self.config.content_path = self.__tmp_dir
self.config.article_path = self.__tmp_dir
self.config.page_path = self.__tmp_dir
self.config.preferred_article_dir = lambda date_created, category: os.path.join(self.config.content_path, category, str(date_created.year), str(date_created.month), str(date_created.day))
def test_custom_generation_works(self):
# the same thing with a specified set of settings should work
settings = read_settings(filename=SAMPLE_CONFIG, override={"PATH": INPUT_PATH, "OUTPUT_PATH": self.temp_path})
pelican = Pelican(settings=settings)
pelican.run()
dcmp = dircmp(self.temp_path, os.sep.join((OUTPUT_PATH, "custom")))
self.assertFilesEqual(recursiveDiff(dcmp))
def setUp (self, override = None):
self.output_path = mkdtemp (prefix = TEST_DIR_PREFIX)
self.content_path = mkdtemp (prefix = TEST_DIR_PREFIX)
settings = {
'PATH': self.content_path,
'OUTPUT_PATH': self.output_path,
'PLUGINS': [linkclass],
'CACKHE_CONTENT': False,
'LINKCLASS_INTERNAL_CLASS': INTERNAL_CLASS,
'LINKCLASS_EXTERNAL_CLASS': EXTERNAL_CLASS
}
if override:
settings.update (override)
## Generate the test Markdown source file
fid = open (os.path.join (self.content_path, '%s.md' % TEST_FILE_STEM),
'w')
fid.write ('''Title: Test
Date:
[%s](%s)
[%s](%s)
[%s](%s)
''' % (INTERNAL_TEXT, INTERNAL_LINK,
EXTERNAL_TEXT, EXTERNAL_LINK_HTTP,
EXTERNAL_TEXT, EXTERNAL_LINK_HTTPS))
fid.close ()
## Run teh Pelican instance
self.settings = read_settings (override = settings)
pelican = Pelican (settings = self.settings)
pelican.run ()
def testKnitrSettings2(self):
settings = read_settings(path=None, override={
'LOAD_CONTENT_CACHE': False,
'PATH': self.contentdir,
'OUTPUT_PATH': self.outputdir,
'RMD_READER_KNITR_OPTS_CHUNK': {'fig.path' : '%s/' % self.figpath},
'RMD_READER_KNITR_OPTS_KNIT': {'progress' : True, 'verbose': True},
'RMD_READER_RENAME_PLOT': True,
'PLUGIN_PATHS': ['../'],
'PLUGINS': ['rmd_reader'],
})
pelican = Pelican(settings=settings)
pelican.run()
outputfilename = os.path.join(self.outputdir,'%s.html' % self.testtitle)
self.assertTrue(os.path.exists(outputfilename),'File %s was not created.' % outputfilename)
imagesdir = os.path.join(self.outputdir, self.figpath)
self.assertTrue(os.path.exists(imagesdir), 'figpath not created.')
imagefile = os.path.join(imagesdir, os.path.splitext(os.path.split(self.contentfile)[1])[0]) + '-1-1.png'
logging.debug(imagefile)
self.assertTrue(os.path.exists(imagefile), 'image correctly named.')
images = glob.glob('%s/*' % imagesdir)
logging.debug(images)
self.assertTrue(len(images) == 1,'Contents of images dir is not correct: %s' % ','.join(images))
def test_sites_generation(self):
'''Test generation of sites with the plugin
Compare with recorded output via ``git diff``.
To generate output for comparison run the command
``pelican -o test_data/output -s test_data/pelicanconf.py \
test_data/content``
Remember to remove the output/ folder before that.
'''
base_path = os.path.dirname(os.path.abspath(__file__))
base_path = os.path.join(base_path, 'test_data')
content_path = os.path.join(base_path, 'content')
output_path = os.path.join(base_path, 'output')
settings_path = os.path.join(base_path, 'pelicanconf.py')
settings = read_settings(path=settings_path, override={
'PATH': content_path,
'OUTPUT_PATH': self.temp_path,
'CACHE_PATH': self.temp_cache,
'PLUGINS': [i18ns],
}
)
pelican = Pelican(settings)
pelican.run()
# compare output
out, err = subprocess.Popen(
['git', 'diff', '--no-ext-diff', '--exit-code', '-w', output_path,
self.temp_path], env={'PAGER': ''},
stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
self.assertFalse(out, 'non-empty `diff` stdout:\n{}'.format(out))
self.assertFalse(err, 'non-empty `diff` stderr:\n{}'.format(out))
def getSettings(self, articles_total_count, articles_list_settings_override):
s = settings.ArticlesListSettings()
s.create_articles_list_directory = MagicMock(return_value=True)
override_settings = {'OUTPUT_PATH': '/output/path'}
pelican_settings = read_settings(override=override_settings)
articles_list_settings = s.get(pelican_settings, articles_total_count)
articles_list_settings.update(articles_list_settings_override)
return articles_list_settings
def __init__(self, settings_file, repo_path=None, images_dir='images', files_dir='files'):
if repo_path is True:
repo_path = os.path.abspath(os.path.dirname(settings_file))
self.repo_path = repo_path
self.images_dir = images_dir
self.files_dir = files_dir
self.settings = read_settings(settings_file)
self.pelican = Pelican(self.settings)
self.article_extensions = tuple([ext for cls in self.article_classes for ext in cls.file_extensions])
def test_custom_generation_works(self):
# the same thing with a specified set of settings should work
with temporary_folder() as temp_path:
pelican = Pelican(path=INPUT_PATH, output_path=temp_path,
settings=read_settings(SAMPLE_CONFIG))
pelican.run()
diff = dircmp(temp_path, os.sep.join((OUTPUT_PATH, "custom")))
self.assertEqual(diff.left_only, [])
self.assertEqual(diff.right_only, [])
self.assertEqual(diff.diff_files, [])
def setUp(self):
self.temp_path = mkdtemp(prefix="pelicanfly.")
pelicanfly_path, _ = os.path.join(os.path.split(pelicanfly.__file__))
self.pelicanfly_static = os.path.join(pelicanfly_path, "static")
self.settings = read_settings(
path=None, override={"PATH": INPUT_PATH, "OUTPUT_PATH": self.temp_path, "PLUGINS": [pelicanfly]}
)
self.pelican = Pelican(self.settings)
mute(True)(self.pelican.run)()
pass
def test_read_empty_settings(self):
# Providing no file should return the default values.
settings = read_settings(None)
expected = copy.deepcopy(DEFAULT_CONFIG)
# Added by configure settings
expected['FEED_DOMAIN'] = ''
expected['ARTICLE_EXCLUDES'] = ['pages']
expected['PAGE_EXCLUDES'] = ['']
self.maxDiff = None
self.assertDictEqual(settings, expected)
def test_deprecated_slug_substitutions_from_file(self):
# This is equivalent to reading a settings file that has
# SLUG_SUBSTITUTIONS defined but no SLUG_REGEX_SUBSTITUTIONS.
settings = read_settings(None, override={
'SLUG_SUBSTITUTIONS': [('C++', 'cpp')]
})
self.assertEqual(settings['SLUG_REGEX_SUBSTITUTIONS'],
[(r'C\+\+', 'cpp')] +
self.settings['SLUG_REGEX_SUBSTITUTIONS'])
self.assertNotIn('SLUG_SUBSTITUTIONS', settings)
def test_custom_generation_works(self):
# the same thing with a specified set of settings should work
settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path,
'LOCALE': locale.normalize('en_US'),
})
pelican = Pelican(settings=settings)
mute(True)(pelican.run)()
self.assertDirsEqual(self.temp_path, os.path.join(OUTPUT_PATH, 'custom'))
def test_read_empty_settings(self):
# Ensure an empty settings file results in default settings.
settings = read_settings(None)
expected = copy.deepcopy(DEFAULT_CONFIG)
# Added by configure settings
expected['FEED_DOMAIN'] = ''
expected['ARTICLE_EXCLUDES'] = ['pages']
expected['PAGE_EXCLUDES'] = ['']
self.maxDiff = None
self.assertDictEqual(settings, expected)
def posterous2fields(api_token, email, password):
"""Imports posterous posts"""
import base64
from datetime import timedelta
try:
# py3k import
import json
except ImportError:
# py2 import
import simplejson as json
try:
# py3k import
import urllib.request as urllib_request
except ImportError:
# py2 import
import urllib2 as urllib_request
def get_posterous_posts(api_token, email, password, page=1):
base64string = base64.encodestring(
("%s:%s" % (email, password)).encode('utf-8')).replace('\n', '')
url = ("http://posterous.com/api/v2/users/me/sites/primary/"
"posts?api_token=%s&page=%d") % (api_token, page)
request = urllib_request.Request(url)
request.add_header('Authorization', 'Basic %s' % base64string.decode())
handle = urllib_request.urlopen(request)
posts = json.loads(handle.read().decode('utf-8'))
return posts
page = 1
posts = get_posterous_posts(api_token, email, password, page)
settings = read_settings()
subs = settings['SLUG_REGEX_SUBSTITUTIONS']
while len(posts) > 0:
posts = get_posterous_posts(api_token, email, password, page)
page += 1
for post in posts:
slug = post.get('slug')
if not slug:
slug = slugify(post.get('title'), regex_subs=subs)
tags = [tag.get('name') for tag in post.get('tags')]
raw_date = post.get('display_date')
date_object = SafeDatetime.strptime(
raw_date[:-6], '%Y/%m/%d %H:%M:%S')
offset = int(raw_date[-5:])
delta = timedelta(hours=(offset / 100))
date_object -= delta
date = date_object.strftime('%Y-%m-%d %H:%M')
kind = 'article' # TODO: Recognise pages
status = 'published' # TODO: Find a way for draft posts
yield (post.get('title'), post.get('body_cleaned'),
slug, date, post.get('user').get('display_name'),
[], tags, status, kind, 'html')
def get_instance(args):
settings = read_settings(args.settings, override=get_config(args))
cls = settings.get('PELICAN_CLASS')
if isinstance(cls, basestring):
module, cls_name = cls.rsplit('.', 1)
module = __import__(module)
cls = getattr(module, cls_name)
return cls(settings)
def test_basic_generation_works(self):
# when running pelican without settings, it should pick up the default
# ones and generate correct output without raising any exception
settings = read_settings(filename=None, override={
'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path,
})
pelican = Pelican(settings=settings)
pelican.run()
dcmp = dircmp(self.temp_path, os.sep.join((OUTPUT_PATH, "basic")))
self.assertFilesEqual(recursiveDiff(dcmp))
def test_custom_generation_works(self):
# the same thing with a specified set of settings should work
settings = read_settings(path=SAMPLE_CONFIG, override={
'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path,
'LOCALE': locale.normalize('en_US'),
})
pelican = Pelican(settings=settings)
pelican.run()
dcmp = dircmp(self.temp_path, os.sep.join((OUTPUT_PATH, "custom")))
self.assertFilesEqual(recursiveDiff(dcmp))
def get_instance(args):
settings = read_settings(args.settings, override=get_config(args))
cls = settings.get("PELICAN_CLASS")
if isinstance(cls, six.string_types):
module, cls_name = cls.rsplit(".", 1)
module = __import__(module)
cls = getattr(module, cls_name)
return cls(settings)
def pelican_generate(files_path):
content_path = os.path.join(files_path, 'content')
conf_path = os.path.join(files_path, 'pelicanconf.py')
output_path = os.path.join(files_path, 'output')
settings = read_settings(conf_path)
pelican = Pelican(
settings=settings,
path=content_path,
output_path=output_path,
theme='notmyidea'
)
pelican.run()
def get_instance(args):
markup = [a.strip().lower() for a in args.markup.split(",")] if args.markup else None
settings = read_settings(args.settings)
cls = settings.get("PELICAN_CLASS")
if isinstance(cls, basestring):
module, cls_name = cls.rsplit(".", 1)
module = __import__(module)
cls = getattr(module, cls_name)
return cls(settings, args.path, args.theme, args.output, markup, args.delete_outputdir)
def test_if_DELETE_OUTPUT_DIRECTORY_is_true_then_raise(self):
"""
WHEN DELETE_OUTPUT_DIRECTORY is True
THEN we'll raise an exception
"""
settings = read_settings(path=None,
override={
'DELETE_OUTPUT_DIRECTORY': True,
'PLUGINS': [pelican_ab]})
pelican = Pelican(settings)
with self.assertRaises(RuntimeError):
pelican.run()
def setUp(self):
self.temp_path = mkdtemp(prefix='pelicanfly.')
pelicanfly_path, _ = os.path.join(os.path.split(pelicanfly.__file__))
self.pelicanfly_static = os.path.join(pelicanfly_path, 'static')
self.settings = read_settings(path=None,
override={
'PATH': INPUT_PATH,
'OUTPUT_PATH': self.temp_path,
'PLUGINS': [pelicanfly]})
self.pelican = Pelican(self.settings)
self.pelican.run()
pass
def test_basic_generation_works(self):
# when running pelican without settings, it should pick up the default
# ones and generate the output without raising any exception / issuing
# any warning.
with temporary_folder() as temp_path:
pelican = Pelican(path=INPUT_PATH, output_path=temp_path)
pelican.run()
# the same thing with a specified set of settins should work
with temporary_folder() as temp_path:
pelican = Pelican(path=INPUT_PATH, output_path=temp_path,
settings=read_settings(SAMPLE_CONFIG))
def main():
parser = argparse.ArgumentParser(description="""A tool to generate a
static blog, with restructured text input files.""")
parser.add_argument(dest='path', nargs='?',
help='Path where to find the content files')
parser.add_argument('-t', '--theme-path', dest='theme',
help='Path where to find the theme templates. If not specified, it will'
'use the default one included with pelican.')
parser.add_argument('-o', '--output', dest='output',
help='Where to output the generated files. If not specified, a directory'
' will be created, named "output" in the current path.')
parser.add_argument('-m', '--markup', default=None, dest='markup',
help='the list of markup language to use (rst or md). Please indicate them'
'separated by commas')
parser.add_argument('-s', '--settings', dest='settings',
help='the settings of the application. Default to None.')
parser.add_argument('-k', '--keep-output-directory', dest='keep',
action='store_true',
help='Keep the output directory and just update all the generated files.'
'Default is to delete the output directory.')
parser.add_argument('--version', action='version', version=VERSION,
help='Print the pelican version and exit')
parser.add_argument('-r', '--autoreload', dest='autoreload', action='store_true',
help="Relaunch pelican each time a modification occurs on the content"
"files")
args = parser.parse_args()
# Split the markup languages only if some have been given. Otherwise, populate
# the variable with None.
markup = [a.strip().lower() for a in args.markup.split(',')] if args.markup else None
if args.settings is None:
settings = {}
settings = read_settings(args.settings)
cls = settings.get('PELICAN_CLASS')
if isinstance(cls, basestring):
module, cls_name = cls.rsplit('.', 1)
module = __import__(module)
cls = getattr(module, cls_name)
pelican = cls(settings, args.path, args.theme, args.output, markup, args.keep)
if args.autoreload:
while True:
try:
if files_changed(pelican.path, pelican.markup):
pelican.run()
except KeyboardInterrupt:
break
else:
pelican.run()
def fields2pelican(fields,
out_markup,
output_path,
dircat=False,
strip_raw=False,
disable_slugs=False,
dirpage=False,
filename_template=None,
filter_author=None,
wp_custpost=False,
wp_attach=False,
attachments=None):
pandoc_version = get_pandoc_version()
posts_require_pandoc = []
settings = read_settings()
slug_subs = settings['SLUG_REGEX_SUBSTITUTIONS']
for (title, content, filename, date, author, categories, tags, status,
kind, in_markup) in fields:
if filter_author and filter_author != author:
continue
if is_pandoc_needed(in_markup) and not pandoc_version:
posts_require_pandoc.append(filename)
slug = not disable_slugs and filename or None
if wp_attach and attachments:
try:
urls = attachments[filename]
links = download_attachments(output_path, urls)
except KeyError:
links = None
else:
links = None
ext = get_ext(out_markup, in_markup)
if ext == '.md':
header = build_markdown_header(title, date, author, categories,
tags, slug, status,
links.values() if links else None)
else:
out_markup = 'rst'
header = build_header(title, date, author, categories, tags, slug,
status,
links.values() if links else None)
out_filename = get_out_filename(output_path, filename, ext, kind,
dirpage, dircat, categories,
wp_custpost, slug_subs)
print(out_filename)
if in_markup in ('html', 'wp-html'):
html_filename = os.path.join(output_path, filename + '.html')
with open(html_filename, 'w', encoding='utf-8') as fp:
# Replace newlines with paragraphs wrapped with <p> so
# HTML is valid before conversion
if in_markup == 'wp-html':
new_content = decode_wp_content(content)
else:
paragraphs = content.splitlines()
paragraphs = ['<p>{0}</p>'.format(p) for p in paragraphs]
new_content = ''.join(paragraphs)
fp.write(new_content)
if pandoc_version < (2, ):
parse_raw = '--parse-raw' if not strip_raw else ''
wrap_none = '--wrap=none' \
if pandoc_version >= (1, 16) else '--no-wrap'
cmd = ('pandoc --normalize {0} --from=html'
' --to={1} {2} -o "{3}" "{4}"')
cmd = cmd.format(parse_raw, out_markup, wrap_none,
out_filename, html_filename)
else:
from_arg = '-f html+raw_html' if not strip_raw else '-f html'
cmd = ('pandoc {0} --to={1}-smart --wrap=none -o "{2}" "{3}"')
cmd = cmd.format(from_arg, out_markup, out_filename,
html_filename)
try:
rc = subprocess.call(cmd, shell=True)
if rc < 0:
error = 'Child was terminated by signal %d' % -rc
exit(error)
elif rc > 0:
error = 'Please, check your Pandoc installation.'
exit(error)
except OSError as e:
error = 'Pandoc execution failed: %s' % e
exit(error)
os.remove(html_filename)
with open(out_filename, 'r', encoding='utf-8') as fs:
content = fs.read()
if out_markup == 'markdown':
# In markdown, to insert a <br />, end a line with two
# or more spaces & then a end-of-line
content = content.replace('\\\n ', ' \n')
content = content.replace('\\\n', ' \n')
if wp_attach and links:
content = update_links_to_attached_files(content, links)
with open(out_filename, 'w', encoding='utf-8') as fs:
fs.write(header + content)
if posts_require_pandoc:
logger.error("Pandoc must be installed to import the following posts:"
"\n {}".format("\n ".join(posts_require_pandoc)))
if wp_attach and attachments and None in attachments:
print("downloading attachments that don't have a parent post")
urls = attachments[None]
download_attachments(output_path, urls)
def test_defaults_not_overwritten(self):
# This assumes 'SITENAME': 'A Pelican Blog'
settings = read_settings(None)
settings['SITENAME'] = 'Not a Pelican Blog'
self.assertNotEqual(settings['SITENAME'], DEFAULT_CONFIG['SITENAME'])
def test_read_empty_settings(self):
"""providing no file should return the default values."""
settings = read_settings(None)
self.assertDictEqual(settings, _DEFAULT_CONFIG)
def _testGet(self, override_settings, expected, articles_total_count):
pelican_settings = read_settings(override=override_settings)
actual = self.settings.get(pelican_settings, articles_total_count)
self.assertEqual(sorted(actual), sorted(expected))
def tearDown(self):
read_settings() # cleanup PYGMENTS_RST_OPTIONS
rmtree(self.temp_path)
rmtree(self.temp_cache)
locale.setlocale(locale.LC_ALL, self.old_locale)
super().tearDown()
def tumblr2fields(api_key, blogname):
""" Imports Tumblr posts (API v2)"""
try:
# py3k import
import json
except ImportError:
# py2 import
import simplejson as json
try:
# py3k import
import urllib.request as urllib_request
except ImportError:
# py2 import
import urllib2 as urllib_request
def get_tumblr_posts(api_key, blogname, offset=0):
url = ("http://api.tumblr.com/v2/blog/%s.tumblr.com/"
"posts?api_key=%s&offset=%d&filter=raw") % (blogname, api_key,
offset)
request = urllib_request.Request(url)
handle = urllib_request.urlopen(request)
posts = json.loads(handle.read().decode('utf-8'))
return posts.get('response').get('posts')
offset = 0
posts = get_tumblr_posts(api_key, blogname, offset)
settings = read_settings()
subs = settings['SLUG_REGEX_SUBSTITUTIONS']
while len(posts) > 0:
for post in posts:
title = \
post.get('title') or \
post.get('source_title') or \
post.get('type').capitalize()
slug = post.get('slug') or slugify(title, regex_subs=subs)
tags = post.get('tags')
timestamp = post.get('timestamp')
date = SafeDatetime.fromtimestamp(
int(timestamp)).strftime("%Y-%m-%d %H:%M:%S")
slug = SafeDatetime.fromtimestamp(
int(timestamp)).strftime("%Y-%m-%d-") + slug
format = post.get('format')
content = post.get('body')
type = post.get('type')
if type == 'photo':
if format == 'markdown':
fmtstr = ''
else:
fmtstr = '<img alt="%s" src="%s" />'
content = ''
for photo in post.get('photos'):
content += '\n'.join(
fmtstr % (photo.get('caption'),
photo.get('original_size').get('url')))
content += '\n\n' + post.get('caption')
elif type == 'quote':
if format == 'markdown':
fmtstr = '\n\n— %s'
else:
fmtstr = '<p>— %s</p>'
content = post.get('text') + fmtstr % post.get('source')
elif type == 'link':
if format == 'markdown':
fmtstr = '[via](%s)\n\n'
else:
fmtstr = '<p><a href="%s">via</a></p>\n'
content = fmtstr % post.get('url') + post.get('description')
elif type == 'audio':
if format == 'markdown':
fmtstr = '[via](%s)\n\n'
else:
fmtstr = '<p><a href="%s">via</a></p>\n'
content = fmtstr % post.get('source_url') + \
post.get('caption') + \
post.get('player')
elif type == 'video':
if format == 'markdown':
fmtstr = '[via](%s)\n\n'
else:
fmtstr = '<p><a href="%s">via</a></p>\n'
source = fmtstr % post.get('source_url')
caption = post.get('caption')
players = '\n'.join(
player.get('embed_code') for player in post.get('player'))
content = source + caption + players
elif type == 'answer':
title = post.get('question')
content = ('<p>'
'<a href="%s" rel="external nofollow">%s</a>'
': %s'
'</p>\n'
' %s' %
(post.get('asking_name'), post.get('asking_url'),
post.get('question'), post.get('answer')))
content = content.rstrip() + '\n'
kind = 'article'
status = 'published' # TODO: Find a way for draft posts
yield (title, content, slug, date, post.get('blog_name'), [type],
tags, status, kind, format)
offset += len(posts)
posts = get_tumblr_posts(api_key, blogname, offset)
def setUp(self):
self.PATH = abspath(dirname(__file__))
default_conf = join(self.PATH, 'default_conf.py')
self.settings = read_settings(default_conf)
def setUp(self):
self.old_locale = locale.setlocale(locale.LC_ALL)
locale.setlocale(locale.LC_ALL, str('C'))
self.PATH = abspath(dirname(__file__))
default_conf = join(self.PATH, 'default_conf.py')
self.settings = read_settings(default_conf)
def main():
parser = argparse.ArgumentParser(description="""A tool to generate a
static blog, with restructured text input files.""")
parser.add_argument(dest='path',
nargs='?',
help='Path where to find the content files')
parser.add_argument(
'-t',
'--theme-path',
dest='theme',
help='Path where to find the theme templates. If not specified, it'
'will use the default one included with pelican.')
parser.add_argument(
'-o',
'--output',
dest='output',
help='Where to output the generated files. If not specified, a directory'
' will be created, named "output" in the current path.')
parser.add_argument(
'-m',
'--markup',
default=None,
dest='markup',
help='the list of markup language to use (rst or md). Please indicate '
'them separated by commas')
parser.add_argument(
'-s',
'--settings',
dest='settings',
default='',
help='the settings of the application. Default to False.')
parser.add_argument('-d',
'--delete-output-directory',
dest='delete_outputdir',
action='store_true',
help='Delete the output directory.')
parser.add_argument('-v',
'--verbose',
action='store_const',
const=log.INFO,
dest='verbosity',
help='Show all messages')
parser.add_argument('-q',
'--quiet',
action='store_const',
const=log.CRITICAL,
dest='verbosity',
help='Show only critical errors')
parser.add_argument('-D',
'--debug',
action='store_const',
const=log.DEBUG,
dest='verbosity',
help='Show all message, including debug messages')
parser.add_argument('--version',
action='version',
version=VERSION,
help='Print the pelican version and exit')
parser.add_argument(
'-r',
'--autoreload',
dest='autoreload',
action='store_true',
help="Relaunch pelican each time a modification occurs on the content"
"files")
args = parser.parse_args()
log.init(args.verbosity)
# Split the markup languages only if some have been given. Otherwise, populate
# the variable with None.
markup = [a.strip().lower()
for a in args.markup.split(',')] if args.markup else None
settings = read_settings(args.settings)
cls = settings.get('PELICAN_CLASS')
if isinstance(cls, basestring):
module, cls_name = cls.rsplit('.', 1)
module = __import__(module)
cls = getattr(module, cls_name)
try:
pelican = cls(settings, args.path, args.theme, args.output, markup,
args.delete_outputdir)
if args.autoreload:
while True:
try:
# Check source dir for changed files ending with the given
# extension in the settings. In the theme dir is no such
# restriction; all files are recursively checked if they
# have changed, no matter what extension the filenames
# have.
if files_changed(pelican.path, pelican.markup) or \
files_changed(pelican.theme, ['']):
pelican.run()
time.sleep(.5) # sleep to avoid cpu load
except KeyboardInterrupt:
break
else:
pelican.run()
except Exception, e:
log.critical(unicode(e))
def dc2fields(file):
"""Opens a Dotclear export file, and yield pelican fields"""
try:
from bs4 import BeautifulSoup
except ImportError:
error = ('Missing dependency '
'"BeautifulSoup4" and "lxml" required '
'to import Dotclear files.')
sys.exit(error)
in_cat = False
in_post = False
category_list = {}
posts = []
with open(file, 'r', encoding='utf-8') as f:
for line in f:
# remove final \n
line = line[:-1]
if line.startswith('[category'):
in_cat = True
elif line.startswith('[post'):
in_post = True
elif in_cat:
fields = line.split('","')
if not line:
in_cat = False
else:
# remove 1st and last ""
fields[0] = fields[0][1:]
# fields[-1] = fields[-1][:-1]
category_list[fields[0]] = fields[2]
elif in_post:
if not line:
in_post = False
break
else:
posts.append(line)
print("%i posts read." % len(posts))
settings = read_settings()
subs = settings['SLUG_REGEX_SUBSTITUTIONS']
for post in posts:
fields = post.split('","')
# post_id = fields[0][1:]
# blog_id = fields[1]
# user_id = fields[2]
cat_id = fields[3]
# post_dt = fields[4]
# post_tz = fields[5]
post_creadt = fields[6]
# post_upddt = fields[7]
# post_password = fields[8]
# post_type = fields[9]
post_format = fields[10]
# post_url = fields[11]
# post_lang = fields[12]
post_title = fields[13]
post_excerpt = fields[14]
post_excerpt_xhtml = fields[15]
post_content = fields[16]
post_content_xhtml = fields[17]
# post_notes = fields[18]
# post_words = fields[19]
# post_status = fields[20]
# post_selected = fields[21]
# post_position = fields[22]
# post_open_comment = fields[23]
# post_open_tb = fields[24]
# nb_comment = fields[25]
# nb_trackback = fields[26]
post_meta = fields[27]
# redirect_url = fields[28][:-1]
# remove seconds
post_creadt = ':'.join(post_creadt.split(':')[0:2])
author = ''
categories = []
tags = []
if cat_id:
categories = [
category_list[id].strip() for id in cat_id.split(',')
]
# Get tags related to a post
tag = (post_meta.replace('{', '').replace('}', '').replace(
'a:1:s:3:\\"tag\\";a:', '').replace('a:0:', ''))
if len(tag) > 1:
if int(len(tag[:1])) == 1:
newtag = tag.split('"')[1]
tags.append(
BeautifulSoup(newtag, 'xml')
# bs4 always outputs UTF-8
.decode('utf-8'))
else:
i = 1
j = 1
while (i <= int(tag[:1])):
newtag = tag.split('"')[j].replace('\\', '')
tags.append(
BeautifulSoup(newtag, 'xml')
# bs4 always outputs UTF-8
.decode('utf-8'))
i = i + 1
if j < int(tag[:1]) * 2:
j = j + 2
"""
dotclear2 does not use markdown by default unless
you use the markdown plugin
Ref: http://plugins.dotaddict.org/dc2/details/formatting-markdown
"""
if post_format == "markdown":
content = post_excerpt + post_content
else:
content = post_excerpt_xhtml + post_content_xhtml
content = content.replace('\\n', '')
post_format = "html"
kind = 'article' # TODO: Recognise pages
status = 'published' # TODO: Find a way for draft posts
yield (post_title, content, slugify(post_title,
regex_subs=subs), post_creadt,
author, categories, tags, status, kind, post_format)
def get_settings(**kwargs):
return read_settings(override={k: v for k, v in kwargs.items()})
def fields2pelican(
content,
output_path,
out_markup,
dircat=False,
strip_raw=False,
disable_slugs=False,
dirpage=False,
filename_template=None,
filter_author=None,
wp_custpost=False,
wp_attach=False,
attachments=None,
):
adapter = PandocAdapter()
settings = read_settings()
slug_subs = settings["SLUG_REGEX_SUBSTITUTIONS"]
# Skip posts not written by a given author
if filter_author and filter_author != content.author:
content.processing_status = ProcessingStatus.SKIPPED
return
# Look for posts that require a markup conversion
if adapter.supports(content.markup) and not adapter.available:
content.processing_status = ProcessingStatus.FAILURE
logger.warning("{} is missing, failed to import: '{}'".format(
adapter.name,
content.slug,
))
return
content.slug = content.filename if not disable_slugs else None
if wp_attach and attachments:
try:
urls = attachments[content.filename]
links = download_attachments(output_path, urls)
except KeyError:
links = None
else:
links = None
renderer_class = RendererFactory.from_markup(content.in_markup, out_markup)
renderer = renderer_class(
content.title,
content.date,
content.author,
content.categories,
content.tags,
content.slug,
content.status,
links.values() if links else None,
)
header = renderer.render()
out_filename = get_out_filename(
output_path,
content.filename,
renderer.file_ext,
content.kind,
dirpage,
dircat,
content.categories,
wp_custpost,
slug_subs,
)
print(out_filename)
output_content = adapter.adapt(
content.in_markup,
out_markup,
output_path,
content.filename,
content.content,
strip_raw,
wp_attach,
links,
out_filename,
)
with open(out_filename, "w", encoding="utf-8") as fs:
fs.write(header + output_content)
content.processing_status = ProcessingStatus.SUCCESS
try:
from urllib.parse import urlparse
except ImportError:
from urlparse import urlparse
from livereload import Server, shell
from pelican import Pelican
from pelican.settings import read_settings
settings = read_settings('pelicanconf.py')
p = Pelican(settings)
def compile():
try:
p.run()
except SystemExit as e:
pass
server = Server()
server.watch(p.settings['PATH'], compile)
server.watch(p.settings['THEME'], compile)
server.watch('./pelicanconf.py', compile)
host_port = p.settings.get('SITEURL') or 'http://localhost:5500'
details = urlparse(host_port)
host, port = details[1].split(':')
server.serve(host=host, port=port, root=settings['OUTPUT_PATH'])
def parse(self):
""" Imports Tumblr posts (API v2)"""
offset = 0
posts = self._get_tumblr_posts(offset)
settings = read_settings()
subs = settings["SLUG_REGEX_SUBSTITUTIONS"]
while len(posts) > 0:
for post in posts:
title = (post.get("title") or post.get("source_title")
or post.get("type").capitalize())
slug = post.get("slug") or slugify(title, regex_subs=subs)
tags = post.get("tags")
timestamp = post.get("timestamp")
date = SafeDatetime.fromtimestamp(
int(timestamp)).strftime("%Y-%m-%d %H:%M:%S")
slug = (SafeDatetime.fromtimestamp(
int(timestamp)).strftime("%Y-%m-%d-") + slug)
format = post.get("format")
content = post.get("body")
type = post.get("type")
if type == "photo":
if format == "markdown":
fmtstr = ""
else:
fmtstr = '<img alt="%s" src="%s" />'
content = ""
for photo in post.get("photos"):
content += "\n".join(fmtstr % (
photo.get("caption"),
photo.get("original_size").get("url"),
))
content += "\n\n" + post.get("caption")
elif type == "quote":
if format == "markdown":
fmtstr = "\n\n— %s"
else:
fmtstr = "<p>— %s</p>"
content = post.get("text") + fmtstr % post.get("source")
elif type == "link":
if format == "markdown":
fmtstr = "[via](%s)\n\n"
else:
fmtstr = '<p><a href="%s">via</a></p>\n'
content = fmtstr % post.get("url") + post.get(
"description")
elif type == "audio":
if format == "markdown":
fmtstr = "[via](%s)\n\n"
else:
fmtstr = '<p><a href="%s">via</a></p>\n'
content = (fmtstr % post.get("source_url") +
post.get("caption") + post.get("player"))
elif type == "video":
if format == "markdown":
fmtstr = "[via](%s)\n\n"
else:
fmtstr = '<p><a href="%s">via</a></p>\n'
source = fmtstr % post.get("source_url")
caption = post.get("caption")
players = "\n".join(
player.get("embed_code")
for player in post.get("player"))
content = source + caption + players
elif type == "answer":
title = post.get("question")
content = ("<p>"
'<a href="%s" rel="external nofollow">%s</a>'
": %s"
"</p>\n"
" %s" % (
post.get("asking_name"),
post.get("asking_url"),
post.get("question"),
post.get("answer"),
))
content = content.rstrip() + "\n"
kind = "article"
status = "published" # TODO: Find a way for draft posts
yield blog2pelican.entities.content.Content(
title=title,
content=content,
slug=slug,
date=date,
author=post.get("blog_name"),
categories=[type],
tags=tags,
status=status,
kind=kind,
markup=format,
)
offset += len(posts)
posts = self._get_tumblr_posts(offset)
