def _parse_metadata(self, meta):
"""Parse and sanitize metadata"""
_DEL = object() # Used as a sentinel
FCNS = {
'tags': lambda x, y: [Tag(t, y) for t in self._to_list(x)] or _DEL,
'date': lambda x, y: get_date(x) if x else _DEL,
'modified': lambda x, y: get_date(x) if x else _DEL,
'category': lambda x, y: Category(x, y) if x else _DEL,
'author': lambda x, y: Author(x, y) if x else _DEL,
'authors':
lambda x, y: [Author(a, y) for a in self._to_list(x)] or _DEL,
'default': lambda x, y: x
}
out = {}
for k, v in meta.items():
k = k.lower()
if k in self.settings['FORMATTED_FIELDS']:
self._md.reset()
temp = self._md.convert("\n".join(self._to_list(v)))
else:
temp = FCNS.get(k, FCNS["default"])(v, self.settings)
if temp is not _DEL:
out[k] = temp
return out
def _parse_metadata(self, meta):
"""Parse and sanitize metadata"""
_DEL = object() # Used as a sentinel
FCNS = {
'tags': lambda x, y: [Tag(t, y) for t in self._to_list(x)] or _DEL,
'date': lambda x, y: get_date(x) if x else _DEL,
'modified': lambda x, y: get_date(x) if x else _DEL,
'category': lambda x, y: Category(x, y) if x else _DEL,
'author': lambda x, y: Author(x, y) if x else _DEL,
'authors': lambda x, y: [Author(a, y) for a in self._to_list(x)]
or _DEL,
'default': lambda x, y: x
}
out = {}
for k, v in meta.items():
k = k.lower()
if k in self.settings['FORMATTED_FIELDS']:
self._md.reset()
temp = self._md.convert("\n".join(self._to_list(v)))
else:
temp = FCNS.get(k, FCNS["default"])(v, self.settings)
if temp is not _DEL:
out[k] = temp
return out
def fix_dates(entries):
"""Convert string star / end dates to datetime objects.
"""
for entry in entries:
if 'startDate' in entry:
entry['startDate'] = get_date(entry['startDate'])
if 'endDate' in entry:
entry['endDate'] = get_date(entry['endDate'])
def dateish(generator):
if 'DATEISH_PROPERTIES' not in generator.settings:
return
for article in generator.articles:
for field in generator.settings['DATEISH_PROPERTIES']:
if hasattr(article, field):
value = getattr(article, field)
if type(value) == list:
setattr(article, field, [get_date(d) for d in value])
else:
setattr(article, field, get_date(value))
def dateish(generator):
if "DATEISH_PROPERTIES" not in generator.settings:
return
for article in generator.articles:
for field in generator.settings["DATEISH_PROPERTIES"]:
if hasattr(article, field):
value = getattr(article, field)
if type(value) == list:
setattr(article, field, [get_date(d) for d in value])
else:
setattr(article, field, get_date(value))
def set_update_date(content):
'''read `update' metadata or filesysystem's mtime
'''
if not content._context:
return
for k, v in content.metadata.items():
if "update" == k.lower():
content.updatedate = get_date(v)
return
if 'UPDATEDATE_MODE' in content.settings and content.settings[
'UPDATEDATE_MODE'] == 'metadata':
return
if 'UPDATE_LOCALE_TO_CREATE_TIME' in content.settings and content.settings[
'UPDATE_LOCALE_TO_CREATE_TIME']:
content.locale_date = datetime.fromtimestamp(
os.path.getctime(content.source_path))
content.locale_date = content.locale_date.replace(microsecond=0)
try:
content.updatedate = datetime.fromtimestamp(
os.path.getmtime(content.source_path))
content.updatedate = content.updatedate.replace(microsecond=0)
#content.date = content.updatedate
except os.error:
logging.error("{} not exists or not readable".format(
content.source_path))
def set_update_date(content):
'''read `update' metadata or filesysystem's mtime
'''
if not content._context:
return
updatedateSet = False
for k, v in content.metadata.items():
if "update" == k.lower():
content.updatedate = get_date(v)
updatedateSet = True
break
if not updatedateSet:
if content.settings.get('UPDATEDATE_MODE', '') == 'metadata':
content.updatedate = content.date
else:
try:
content.updatedate = datetime.fromtimestamp(os.path.getmtime(content.source_path))
content.updatedate = content.updatedate.replace(microsecond = 0)
except os.error:
logging.error("{} not exists or not readable".format(content.source_path))
if hasattr(content, 'date') and content.date.tzinfo is not None:
content.updatedate = set_date_tzinfo(content.updatedate, content.date.tzinfo.zone)
def get_date_modified(self, page, default):
if hasattr(page, 'modified'):
if isinstance(page.modified, datetime):
return page.modified
return get_date(page.modified)
else:
return default
def get_date_modified(self, page, default):
if hasattr(page, 'modified'):
if isinstance(page.modified, datetime):
return page.modified
return get_date(page.modified)
else:
return default
def set_update_date(content):
'''read `update' metadata or filesysystem's mtime
'''
if not content._context:
return
updatedateSet = False
for k, v in content.metadata.items():
if "update" == k.lower():
content.updatedate = get_date(v)
updatedateSet = True
break
if not updatedateSet:
if content.settings.get('UPDATEDATE_MODE', '') == 'metadata':
content.updatedate = content.date
else:
try:
content.updatedate = datetime.fromtimestamp(
os.path.getmtime(content.source_path))
content.updatedate = content.updatedate.replace(microsecond=0)
except os.error:
logging.error("{} not exists or not readable".format(
content.source_path))
if hasattr(content, 'date') and content.date.tzinfo is not None:
content.updatedate = set_date_tzinfo(content.updatedate,
content.date.tzinfo.zone)
def test_get_date(self):
# valid ones
date = datetime.datetime(year=2012, month=11, day=22)
date_hour = datetime.datetime(year=2012, month=11, day=22, hour=22,
minute=11)
date_hour_sec = datetime.datetime(year=2012, month=11, day=22, hour=22,
minute=11, second=10)
dates = {'2012-11-22': date,
'2012/11/22': date,
'2012-11-22 22:11': date_hour,
'2012/11/22 22:11': date_hour,
'22-11-2012': date,
'22/11/2012': date,
'22.11.2012': date,
'2012-22-11': date,
'22.11.2012 22:11': date_hour,
'2012-11-22 22:11:10': date_hour_sec}
for value, expected in dates.items():
self.assertEquals(utils.get_date(value), expected, value)
# invalid ones
invalid_dates = ('2010-110-12', 'yay')
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def __init__(self, *args, **kwargs):
super(Session, self).__init__(*args, **kwargs)
has_duration = hasattr(self, 'duration')
has_start_date = hasattr(self, 'start_date')
if has_duration:
d = self.duration.split(' ')
if len(d) <> 2 or d[1] <> "minutes":
logger.error("Unknown duration format: %s", self.duration)
self.duration = timedelta(minutes=int(d[0]))
if has_start_date:
self.start_date = get_date(self.start_date)
self.locale_start_date = strftime(self.start_date, "%A %d")
self.locale_start_time = strftime(self.start_date, "%H:%M")
if has_duration and has_start_date:
self.end_date = self.start_date + self.duration
self.locale_end_time = strftime(self.end_date, "%H:%M")
if not hasattr(self, 'bios'):
bios = conference.bios.by_role_and_slug['speaker']
self.bios = []
for speaker in self.speakers:
slug = slugify(speaker)
self.bios.append(slug)
if slug not in bios:
bio = Bio("", {'title': speaker}, settings=self.settings,
source_path="", context=self._context)
conference.add_bio(bio)
def read(self, filename):
with pelican_open(filename) as text:
md_slides = text.split('\n---\n')
slides = []
# Process each slide separately.
for md_slide in md_slides:
slide = {}
sections = md_slide.split('\n\n')
# Extract metadata at the beginning of the slide (look for key: value)
# pairs.
metadata_section = sections[0]
metadata = self.parse_metadata(metadata_section)
slide.update(metadata)
remainder_index = metadata and 1 or 0
# Get the content from the rest of the slide.
content_section = '\n\n'.join(sections[remainder_index:])
html = markdown.markdown(content_section,
extensions=self.extensions)
slide['content'] = self.postprocess_html(html, metadata)
slides.append(slide)
metadata = {
'slides': slides,
'title': slides[0]['title'],
'date': get_date(slides[0]['date']),
'template': 'slide'
}
return "", metadata
def test_get_date(self):
# valid ones
date = datetime.datetime(year=2012, month=11, day=22)
date_hour = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11)
date_hour_sec = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10)
dates = {
'2012-11-22': date,
'2012/11/22': date,
'2012-11-22 22:11': date_hour,
'2012/11/22 22:11': date_hour,
'22-11-2012': date,
'22/11/2012': date,
'22.11.2012': date,
'2012-22-11': date,
'22.11.2012 22:11': date_hour,
'2012-11-22 22:11:10': date_hour_sec
}
for value, expected in dates.items():
self.assertEquals(utils.get_date(value), expected, value)
# invalid ones
invalid_dates = ('2010-110-12', 'yay')
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def get_entry_datetime(entry):
try:
return utils.get_date(entry.get('published', ''))
except ValueError:
warning('Webring Plugin: Invalid date on feed entry titled "%s"' %
entry.get('title', 'Unknown title'))
return utils.SafeDatetime.now()
def _parse_date(obj):
"""Return a string representing a date"""
# If it's already a date object, make it a string so Pelican can parse it
# and make sure it has a timezone
if isinstance(obj, datetime.date):
obj = obj.isoformat()
return get_date(str(obj).strip().replace('_', ' '))
def dateish(generator):
if 'DATEISH_PROPERTIES' not in generator.settings:
return
for article in generator.articles:
for field in generator.settings['DATEISH_PROPERTIES']:
if hasattr(article, field):
text = getattr(article, field)
as_datetime = get_date(text)
setattr(article, field, as_datetime)
def get_entry_datetime(key, entry):
try:
# Empty string raises ValueError in dateutil.parser.parse()
return utils.get_date(entry.get(key, ""))
except ValueError:
warning(
"Webring Plugin: Invalid '%s' on feed entry titled '%s'",
key,
entry.get("title", "Unknown title"),
)
return None
def read(self, source_path):
self._source_path = source_path
self._md = Markdown(**self.settings["MARKDOWN"])
if "Lectures" in source_path:
self._md.preprocessors.register(
BlockquotesPreprocessor(self._md), "blockquotes", 10
)
with pelican_open(source_path) as text:
content = self._md.convert(text)
if hasattr(self._md, "Meta"):
metadata = self._parse_metadata(self._md.Meta)
else:
metadata = {}
# Add the TOC to the metadata.
if len(self._md.toc) > 300:
metadata["table_of_contents"] = self._md.toc
# Get the title from the first h1
if "title" not in metadata and len(self._md.toc_tokens):
first_title = self._md.toc_tokens[0]
metadata["title"] = first_title["name"]
content = content.replace(
'<h1 id="{id}">{name}</h1>'.format(**first_title), ""
)
# Get the date from the filename, if possible.
parts = os.path.splitext(os.path.basename(source_path))[0].split("-")
if "read_on" in metadata:
metadata["date"] = datetime.strptime(metadata["read_on"], "%B %Y")
elif len(parts) >= 3:
metadata["date"] = get_date("-".join(parts[:3]))
if "slug" not in metadata:
metadata["slug"] = slugify(
metadata["title"], self.settings.get("SLUG_REGEX_SUBSTITUTIONS", [])
)
category = os.path.basename(
os.path.abspath(os.path.join(source_path, os.pardir))
)
metadata["category"] = self.process_metadata("category", category)
return content, metadata
def test_get_date(self):
# valid ones
date = datetime.datetime(year=2012, month=11, day=22)
date_hour = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11)
date_hour_sec = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10)
date_hour_sec_z = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10,
tzinfo=pytz.timezone('UTC'))
date_hour_sec_est = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10,
tzinfo=pytz.timezone('EST'))
date_hour_sec_frac_z = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10,
microsecond=123000, tzinfo=pytz.timezone('UTC'))
dates = {
'2012-11-22': date,
'2012/11/22': date,
'2012-11-22 22:11': date_hour,
'2012/11/22 22:11': date_hour,
'22-11-2012': date,
'22/11/2012': date,
'22.11.2012': date,
'22.11.2012 22:11': date_hour,
'2012-11-22 22:11:10': date_hour_sec,
'2012-11-22T22:11:10Z': date_hour_sec_z,
'2012-11-22T22:11:10-0500': date_hour_sec_est,
'2012-11-22T22:11:10.123Z': date_hour_sec_frac_z,
}
# invalid ones
invalid_dates = ['2010-110-12', 'yay']
if version_info < (3, 2):
dates.pop('2012-11-22T22:11:10-0500')
invalid_dates.append('2012-11-22T22:11:10-0500')
for value, expected in dates.items():
self.assertEqual(utils.get_date(value), expected, value)
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def default_metadata(settings=None, process=None):
metadata = {}
if settings:
for name, value in dict(settings.get('DEFAULT_METADATA', {})).items():
if process:
value = process(name, value)
metadata[name] = value
if 'DEFAULT_CATEGORY' in settings:
value = settings['DEFAULT_CATEGORY']
if process:
value = process('category', value)
metadata['category'] = value
if settings.get('DEFAULT_DATE', None) and \
settings['DEFAULT_DATE'] != 'fs':
if isinstance(settings['DEFAULT_DATE'], six.string_types):
metadata['date'] = get_date(settings['DEFAULT_DATE'])
else:
metadata['date'] = SafeDatetime(*settings['DEFAULT_DATE'])
return metadata
def default_metadata(settings=None, process=None):
metadata = {}
if settings:
for name, value in dict(settings.get('DEFAULT_METADATA', {})).items():
if process:
value = process(name, value)
metadata[name] = value
if 'DEFAULT_CATEGORY' in settings:
value = settings['DEFAULT_CATEGORY']
if process:
value = process('category', value)
metadata['category'] = value
if settings.get('DEFAULT_DATE', None) and \
settings['DEFAULT_DATE'] != 'fs':
if isinstance(settings['DEFAULT_DATE'], six.string_types):
metadata['date'] = get_date(settings['DEFAULT_DATE'])
else:
metadata['date'] = SafeDatetime(*settings['DEFAULT_DATE'])
return metadata
def duplicate_on_dates(generator):
"""
Articles (events) with `dates` property are recurring. Create a
copy of the article for each date in `dates`.
"""
articles = []
for article in generator.articles:
if not hasattr(article, 'dates'):
articles.append(article)
continue
log.debug('Event {} has {} occurrences.'.format(article.get_relative_source_path(), len(article.dates)))
for i, date in enumerate(article.dates, 2):
event = copy(article)
articles.append(event)
event.slug += '--' + str(i) # Create hopefully unique slug
# The comment following '#' can be anything (e.g. visitor count)
date, _, event.dates_comment = date.partition('#')
# From pelican.contents.Content.__init__
timezone = getattr(event, 'timezone', event.settings.get('TIMEZONE', 'UTC'))
event.date = set_date_tzinfo(get_date(date), timezone)
event.locale_date = strftime(event.date, event.date_format)
articles.sort(key=attrgetter(generator.settings['ARTICLE_ORDER_BY']), reverse=True)
generator.articles = articles
from docutils import core
from markdown import Markdown
import re
import string
# import the directives to have pygments support
import rstdirectives
from pelican.utils import get_date, open
_METADATAS_PROCESSORS = {
'tags': lambda x: map(string.strip, x.split(',')),
'date': lambda x: get_date(x),
'status': string.strip,
}
class RstReader(object):
def _parse_metadata(self, content):
"""Return the dict containing metadatas"""
output = {}
for m in re.compile('^:([a-z]+): (.*)\s', re.M).finditer(content):
name, value = m.group(1).lower(), m.group(2)
output[name] = _METADATAS_PROCESSORS.get(
name, lambda x:x
)(value)
return output
def read(self, filename):
def test_get_date(self):
# valid ones
date = utils.SafeDatetime(year=2012, month=11, day=22)
date_hour = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11)
date_hour_z = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
tzinfo=pytz.timezone('UTC'))
date_hour_est = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
tzinfo=pytz.timezone('EST'))
date_hour_sec = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10)
date_hour_sec_z = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10,
tzinfo=pytz.timezone('UTC'))
date_hour_sec_est = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10,
tzinfo=pytz.timezone('EST'))
date_hour_sec_frac_z = utils.SafeDatetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10,
microsecond=123000,
tzinfo=pytz.timezone('UTC'))
dates = {
'2012-11-22': date,
'2012/11/22': date,
'2012-11-22 22:11': date_hour,
'2012/11/22 22:11': date_hour,
'22-11-2012': date,
'22/11/2012': date,
'22.11.2012': date,
'22.11.2012 22:11': date_hour,
'2012-11-22T22:11Z': date_hour_z,
'2012-11-22T22:11-0500': date_hour_est,
'2012-11-22 22:11:10': date_hour_sec,
'2012-11-22T22:11:10Z': date_hour_sec_z,
'2012-11-22T22:11:10-0500': date_hour_sec_est,
'2012-11-22T22:11:10.123Z': date_hour_sec_frac_z,
}
# examples from http://www.w3.org/TR/NOTE-datetime
iso_8601_date = utils.SafeDatetime(year=1997, month=7, day=16)
iso_8601_date_hour_tz = utils.SafeDatetime(year=1997,
month=7,
day=16,
hour=19,
minute=20,
tzinfo=pytz.timezone('CET'))
iso_8601_date_hour_sec_tz = utils.SafeDatetime(
year=1997,
month=7,
day=16,
hour=19,
minute=20,
second=30,
tzinfo=pytz.timezone('CET'))
iso_8601_date_hour_sec_ms_tz = utils.SafeDatetime(
year=1997,
month=7,
day=16,
hour=19,
minute=20,
second=30,
microsecond=450000,
tzinfo=pytz.timezone('CET'))
iso_8601 = {
'1997-07-16': iso_8601_date,
'1997-07-16T19:20+01:00': iso_8601_date_hour_tz,
'1997-07-16T19:20:30+01:00': iso_8601_date_hour_sec_tz,
'1997-07-16T19:20:30.45+01:00': iso_8601_date_hour_sec_ms_tz,
}
# invalid ones
invalid_dates = ['2010-110-12', 'yay']
for value, expected in dates.items():
self.assertEqual(utils.get_date(value), expected, value)
for value, expected in iso_8601.items():
self.assertEqual(utils.get_date(value), expected, value)
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def test_get_date(self):
# valid ones
date = datetime.datetime(year=2012, month=11, day=22)
date_hour = datetime.datetime(year=2012, month=11, day=22, hour=22, minute=11)
date_hour_z = datetime.datetime(year=2012, month=11, day=22, hour=22, minute=11, tzinfo=pytz.timezone("UTC"))
date_hour_est = datetime.datetime(year=2012, month=11, day=22, hour=22, minute=11, tzinfo=pytz.timezone("EST"))
date_hour_sec = datetime.datetime(year=2012, month=11, day=22, hour=22, minute=11, second=10)
date_hour_sec_z = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10, tzinfo=pytz.timezone("UTC")
)
date_hour_sec_est = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10, tzinfo=pytz.timezone("EST")
)
date_hour_sec_frac_z = datetime.datetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10, microsecond=123000, tzinfo=pytz.timezone("UTC")
)
dates = {
"2012-11-22": date,
"2012/11/22": date,
"2012-11-22 22:11": date_hour,
"2012/11/22 22:11": date_hour,
"22-11-2012": date,
"22/11/2012": date,
"22.11.2012": date,
"22.11.2012 22:11": date_hour,
"2012-11-22T22:11Z": date_hour_z,
"2012-11-22T22:11-0500": date_hour_est,
"2012-11-22 22:11:10": date_hour_sec,
"2012-11-22T22:11:10Z": date_hour_sec_z,
"2012-11-22T22:11:10-0500": date_hour_sec_est,
"2012-11-22T22:11:10.123Z": date_hour_sec_frac_z,
}
# examples from http://www.w3.org/TR/NOTE-datetime
iso_8601_date = datetime.datetime(year=1997, month=7, day=16)
iso_8601_date_hour_tz = datetime.datetime(
year=1997, month=7, day=16, hour=19, minute=20, tzinfo=pytz.timezone("CET")
)
iso_8601_date_hour_sec_tz = datetime.datetime(
year=1997, month=7, day=16, hour=19, minute=20, second=30, tzinfo=pytz.timezone("CET")
)
iso_8601_date_hour_sec_ms_tz = datetime.datetime(
year=1997, month=7, day=16, hour=19, minute=20, second=30, microsecond=450000, tzinfo=pytz.timezone("CET")
)
iso_8601 = {
"1997-07-16": iso_8601_date,
"1997-07-16T19:20+01:00": iso_8601_date_hour_tz,
"1997-07-16T19:20:30+01:00": iso_8601_date_hour_sec_tz,
"1997-07-16T19:20:30.45+01:00": iso_8601_date_hour_sec_ms_tz,
}
# invalid ones
invalid_dates = ["2010-110-12", "yay"]
for value, expected in dates.items():
self.assertEqual(utils.get_date(value), expected, value)
for value, expected in iso_8601.items():
self.assertEqual(utils.get_date(value), expected, value)
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def format_date(value, date_format=DEFAULT_DATE_FORMAT):
return strftime(get_date(value), date_format)
try:
from markdown import Markdown
except ImportError:
Markdown = False # NOQA
# Metadata processors have no way to discard an unwanted value, so we have
# them return this value instead to signal that it should be discarded later.
# This means that _filter_discardable_metadata() must be called on processed
# metadata dicts before use, to remove the items with the special value.
_DISCARD = object()
METADATA_PROCESSORS = {
'tags':
lambda x, y:
([Tag(tag, y) for tag in ensure_metadata_list(x)] or _DISCARD),
'date':
lambda x, y: get_date(x.replace('_', ' ')),
'modified':
lambda x, y: get_date(x),
'status':
lambda x, y: x.strip() or _DISCARD,
'category':
lambda x, y: _process_if_nonempty(Category, x, y),
'author':
lambda x, y: _process_if_nonempty(Author, x, y),
'authors':
lambda x, y:
([Author(author, y) for author in ensure_metadata_list(x)] or _DISCARD),
'slug':
lambda x, y: x.strip() or _DISCARD,
}
asciidoc = False
try:
from html import escape
except ImportError:
from cgi import escape
try:
from html.parser import HTMLParser
except ImportError:
from HTMLParser import HTMLParser
from pelican.contents import Category, Tag, Author
from pelican.utils import get_date, pelican_open
METADATA_PROCESSORS = {
'tags': lambda x, y: [Tag(tag, y) for tag in x.split(',')],
'date': lambda x, y: get_date(x),
'status': lambda x, y: x.strip(),
'category': Category,
'author': Author,
}
class Reader(object):
enabled = True
file_extensions = ['static']
extensions = None
def __init__(self, settings):
self.settings = settings
def process_metadata(self, name, value):
except ImportError:
Markdown = False # NOQA
try:
from html import escape
except ImportError:
from cgi import escape
from six.moves.html_parser import HTMLParser
from pelican import signals
from pelican.contents import Page, Category, Tag, Author
from pelican.utils import get_date, pelican_open, FileStampDataCacher, SafeDatetime
METADATA_PROCESSORS = {
"tags": lambda x, y: [Tag(tag, y) for tag in x.split(",")],
"date": lambda x, y: get_date(x),
"modified": lambda x, y: get_date(x),
"status": lambda x, y: x.strip(),
"category": Category,
"author": Author,
"authors": lambda x, y: [Author(author.strip(), y) for author in x.split(",")],
}
logger = logging.getLogger(__name__)
class BaseReader(object):
"""Base class to read files.
This class is used to process static files, and it can be inherited for
other types of file. A Reader class must have the following attributes:
# -*- coding: utf-8 -*-
from docutils import core
from markdown import Markdown
import re
import string
# import the directives to have pygments support
import rstdirectives
from pelican.utils import get_date, open
_METADATAS_PROCESSORS = {
"tags": lambda x: map(string.strip, x.split(",")),
"date": lambda x: get_date(x),
"status": string.strip,
}
class RstReader(object):
def _parse_metadata(self, content):
"""Return the dict containing metadatas"""
output = {}
for m in re.compile("^:([a-z]+): (.*)\s", re.M).finditer(content):
name, value = m.group(1).lower(), m.group(2)
output[name] = _METADATAS_PROCESSORS.get(name, lambda x: x)(value)
return output
def read(self, filename):
"""Parse restructured text"""
text = open(filename)
def test_get_date(self):
# valid ones
date = utils.SafeDatetime(year=2012, month=11, day=22)
date_hour = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11)
date_hour_z = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11,
tzinfo=pytz.timezone('UTC'))
date_hour_est = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11,
tzinfo=pytz.timezone('EST'))
date_hour_sec = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10)
date_hour_sec_z = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10,
tzinfo=pytz.timezone('UTC'))
date_hour_sec_est = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10,
tzinfo=pytz.timezone('EST'))
date_hour_sec_frac_z = utils.SafeDatetime(
year=2012, month=11, day=22, hour=22, minute=11, second=10,
microsecond=123000, tzinfo=pytz.timezone('UTC'))
dates = {
'2012-11-22': date,
'2012/11/22': date,
'2012-11-22 22:11': date_hour,
'2012/11/22 22:11': date_hour,
'22-11-2012': date,
'22/11/2012': date,
'22.11.2012': date,
'22.11.2012 22:11': date_hour,
'2012-11-22T22:11Z': date_hour_z,
'2012-11-22T22:11-0500': date_hour_est,
'2012-11-22 22:11:10': date_hour_sec,
'2012-11-22T22:11:10Z': date_hour_sec_z,
'2012-11-22T22:11:10-0500': date_hour_sec_est,
'2012-11-22T22:11:10.123Z': date_hour_sec_frac_z,
}
# examples from http://www.w3.org/TR/NOTE-datetime
iso_8601_date = utils.SafeDatetime(year=1997, month=7, day=16)
iso_8601_date_hour_tz = utils.SafeDatetime(
year=1997, month=7, day=16, hour=19, minute=20,
tzinfo=pytz.timezone('CET'))
iso_8601_date_hour_sec_tz = utils.SafeDatetime(
year=1997, month=7, day=16, hour=19, minute=20, second=30,
tzinfo=pytz.timezone('CET'))
iso_8601_date_hour_sec_ms_tz = utils.SafeDatetime(
year=1997, month=7, day=16, hour=19, minute=20, second=30,
microsecond=450000, tzinfo=pytz.timezone('CET'))
iso_8601 = {
'1997-07-16': iso_8601_date,
'1997-07-16T19:20+01:00': iso_8601_date_hour_tz,
'1997-07-16T19:20:30+01:00': iso_8601_date_hour_sec_tz,
'1997-07-16T19:20:30.45+01:00': iso_8601_date_hour_sec_ms_tz,
}
# invalid ones
invalid_dates = ['2010-110-12', 'yay']
for value, expected in dates.items():
self.assertEqual(utils.get_date(value), expected, value)
for value, expected in iso_8601.items():
self.assertEqual(utils.get_date(value), expected, value)
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def str2defaultformat(data):
return strftime(get_date(data), DEFAULT_DATE_FORMAT)
def add_metadata_processors(arg):
METADATA_PROCESSORS['end_date'] = METADATA_PROCESSORS.get(
'end_date', METADATA_PROCESSORS.get('date'))
METADATA_PROCESSORS['end_date'] = lambda x, y: get_date(x.replace(
'_', ' '))
def get_date_modified(self, page, defalut):
if hasattr(page, 'modified'):
return get_date(getattr(page, 'modified'))
else:
return defalut
def str2defaultformat(data):
return strftime(get_date(data), DEFAULT_DATE_FORMAT)
except ImportError:
Markdown = False # NOQA
try:
from html import escape
except ImportError:
from cgi import escape
from six.moves.html_parser import HTMLParser
from pelican import signals
from pelican.contents import Page, Category, Tag, Author
from pelican.utils import get_date, pelican_open, FileStampDataCacher, SafeDatetime
METADATA_PROCESSORS = {
'tags': lambda x, y: [Tag(tag, y) for tag in x.split(',')],
'date': lambda x, y: get_date(x),
'modified': lambda x, y: get_date(x),
'status': lambda x, y: x.strip(),
'category': Category,
'author': Author,
'authors': lambda x, y: [Author(author.strip(), y) for author in x.split(',')],
}
logger = logging.getLogger(__name__)
class BaseReader(object):
"""Base class to read files.
This class is used to process static files, and it can be inherited for
other types of file. A Reader class must have the following attributes:
'modified': False,
'status': False,
'category': False,
'author': False,
'save_as': False,
'url': False,
'authors': False,
'slug': False
}
METADATA_PROCESSORS = {
'tags': lambda x, y: ([
Tag(tag, y)
for tag in ensure_metadata_list(x)
] or _DISCARD),
'date': lambda x, y: get_date(x.replace('_', ' ')),
'modified': lambda x, y: get_date(x),
'status': lambda x, y: x.strip() or _DISCARD,
'category': lambda x, y: _process_if_nonempty(Category, x, y),
'author': lambda x, y: _process_if_nonempty(Author, x, y),
'authors': lambda x, y: ([
Author(author, y)
for author in ensure_metadata_list(x)
] or _DISCARD),
'slug': lambda x, y: x.strip() or _DISCARD,
}
logger = logging.getLogger(__name__)
def ensure_metadata_list(text):
# import the directives to have pygments support
import rstdirectives
except ImportError:
core = False
try:
from markdown import Markdown
except ImportError:
Markdown = False
import re
import string
from pelican.utils import get_date, open
_METADATAS_PROCESSORS = {
'tags': lambda x: map(string.strip, x.split(',')),
'date': lambda x: get_date(x),
'status': string.strip,
}
class Reader(object):
enabled = True
class RstReader(Reader):
enabled = bool(core)
extension = "rst"
def _parse_metadata(self, content):
"""Return the dict containing metadatas"""
output = {}
from pelican import rstdirectives # NOQA
except ImportError:
core = False
try:
from markdown import Markdown
except ImportError:
Markdown = False # NOQA
import re
from pelican.contents import Category, Tag, Author
from pelican.utils import get_date, open
_METADATA_PROCESSORS = {
'tags': lambda x, y: [Tag(tag, y) for tag in unicode(x).split(',')],
'date': lambda x, y: get_date(x),
'status': lambda x, y: unicode.strip(x),
'category': Category,
'author': Author,
}
class Reader(object):
enabled = True
extensions = None
def __init__(self, settings):
self.settings = settings
def process_metadata(self, name, value):
if name in _METADATA_PROCESSORS:
except ImportError:
from cgi import escape
try:
from html.parser import HTMLParser
except ImportError:
from HTMLParser import HTMLParser
from pelican.contents import Page, Category, Tag, Author
from pelican.utils import get_date, pelican_open
logger = logging.getLogger(__name__)
METADATA_PROCESSORS = {
"tags": lambda x, y: [Tag(tag, y) for tag in x.split(",")],
"date": lambda x, y: get_date(x),
"status": lambda x, y: x.strip(),
"category": Category,
"author": Author,
}
class Reader(object):
enabled = True
file_extensions = ["static"]
extensions = None
def __init__(self, settings):
self.settings = settings
def process_metadata(self, name, value):
from pelican import rstdirectives # NOQA
except ImportError:
core = False
try:
from markdown import Markdown
except ImportError:
Markdown = False # NOQA
import re
from pelican.contents import Category, Tag, Author
from pelican.utils import get_date, open
_METADATA_PROCESSORS = {
'tags': lambda x, y, z: [Tag(tag, y, z) for tag in unicode(x).split(',')],
'date': lambda x, y, z: get_date(x),
'status': lambda x, y, z: unicode.strip(x),
'category': Category,
'author': Author,
}
class Reader(object):
enabled = True
extensions = None
def __init__(self, settings):
self.settings = settings
self.default_lang = getattr(settings, 'DEFAULT_LANG', '').lower()
def process_metadata(self, metadata):
def test_get_date_returns_correct_value(self):
date = utils.SafeDatetime(year=2020, month=0o4, day=30)
date_hour = utils.SafeDatetime(year=2020,
month=0o4,
day=30,
hour=20,
minute=10)
date_hour_utc = utils.SafeDatetime(year=2020,
month=0o4,
day=30,
hour=20,
minute=10,
tzinfo=pytz.timezone('UTC'))
date_hour_est = utils.SafeDatetime(year=2020,
month=0o4,
day=30,
hour=20,
minute=10,
tzinfo=pytz.timezone('EST'))
date_hour_seconds = utils.SafeDatetime(
year=2020,
month=0o4,
day=30,
hour=20,
minute=10,
second=20,
)
date_hour_seconds_utc = utils.SafeDatetime(year=2020,
month=0o4,
day=30,
hour=20,
minute=10,
second=20,
tzinfo=pytz.timezone('UTC'))
date_hour_seconds_est = utils.SafeDatetime(year=2020,
month=0o4,
day=30,
hour=20,
minute=10,
second=20,
tzinfo=pytz.timezone('EST'))
date_hour_seconds_usecs_utc = utils.SafeDatetime(
year=2020,
month=0o4,
day=30,
hour=20,
minute=10,
second=20,
microsecond=123000,
tzinfo=pytz.timezone('UTC'))
dates = {
'2020/04/30': date,
'2020-04-30': date,
'30/04/2020': date,
'30-04-2020': date,
'30.04.2020': date,
'2020/04/30 20:10': date_hour,
'2020-04-30 20:10': date_hour,
'30.04.2020 20:10': date_hour,
'2020-04-30T20:10Z': date_hour_utc,
'2020-04-30T20:10-0500': date_hour_est,
'2020/04/30 20:10:20': date_hour_seconds,
'2020-04-30T20:10:20Z': date_hour_seconds_utc,
'2020/04/30T20:10:20-0500': date_hour_seconds_est,
'2020-04-30T20:10:20.123Z': date_hour_seconds_usecs_utc,
}
# ISO 8601 datetime format
iso_8601_date = utils.SafeDatetime(
year=1997,
month=7,
day=15,
)
iso_8601_date_hour_timezone = utils.SafeDatetime(
year=1997,
month=7,
day=15,
hour=19,
minute=20,
tzinfo=pytz.timezone('CET'))
iso_8601_date_hour_secs_timezone = utils.SafeDatetime(
year=1997,
month=7,
day=15,
hour=19,
minute=20,
second=30,
tzinfo=pytz.timezone('CET'))
iso_8601_date_hour_secs_ms = utils.SafeDatetime(
year=1997,
month=7,
day=15,
hour=19,
minute=20,
second=30,
microsecond=120000,
tzinfo=pytz.timezone('CET'))
iso_8601_format = {
'1997-07-15': iso_8601_date,
'1997-07-15T19:20+01:00': iso_8601_date_hour_timezone,
'1997-07-15T19:20:30+01:00': iso_8601_date_hour_secs_timezone,
'1997-07-15T19:20:30.12+01:00': iso_8601_date_hour_secs_ms,
}
# Invalid dates
invalid_dates = ['2040-123-3,' 'wrongdate', '2.12.ac', '001/1001']
for value, expected in dates.items():
self.assertEqual(utils.get_date(value), expected, value)
self.assertTrue(utils.get_date(value) is not None)
for value, expected in iso_8601_format.items():
self.assertEqual(utils.get_date(value), expected, value)
for date in invalid_dates:
self.assertRaises(ValueError, utils.get_date, date)
def test_get_date(self):
# valid ones
date = datetime.datetime(year=2012, month=11, day=22)
date_hour = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11)
date_hour_sec = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10)
date_hour_sec_z = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10,
tzinfo=pytz.timezone('UTC'))
date_hour_sec_est = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10,
tzinfo=pytz.timezone('EST'))
date_hour_sec_frac_z = datetime.datetime(year=2012,
month=11,
day=22,
hour=22,
minute=11,
second=10,
microsecond=123000,
tzinfo=pytz.timezone('UTC'))
dates = {
'2012-11-22': date,
'2012/11/22': date,
'2012-11-22 22:11': date_hour,
'2012/11/22 22:11': date_hour,
'22-11-2012': date,
'22/11/2012': date,
'22.11.2012': date,
'22.11.2012 22:11': date_hour,
'2012-11-22 22:11:10': date_hour_sec,
'2012-11-22T22:11:10Z': date_hour_sec_z,
'2012-11-22T22:11:10-0500': date_hour_sec_est,
'2012-11-22T22:11:10.123Z': date_hour_sec_frac_z,
}
# invalid ones
invalid_dates = ['2010-110-12', 'yay']
if version_info < (3, 2):
dates.pop('2012-11-22T22:11:10-0500')
invalid_dates.append('2012-11-22T22:11:10-0500')
for value, expected in dates.items():
self.assertEqual(utils.get_date(value), expected, value)
for item in invalid_dates:
self.assertRaises(ValueError, utils.get_date, item)
def format_date(value, date_format=DEFAULT_DATE_FORMAT):
return strftime(get_date(value), date_format)
'modified': False,
'status': False,
'category': False,
'author': False,
'save_as': False,
'url': False,
'authors': False,
'slug': False
}
METADATA_PROCESSORS = {
'tags': lambda x, y: ([
Tag(tag, y)
for tag in ensure_metadata_list(x)
] or _DISCARD),
'date': lambda x, y: get_date(x.replace('_', ' ')),
'modified': lambda x, y: get_date(x),
'status': lambda x, y: x.strip() or _DISCARD,
'category': lambda x, y: _process_if_nonempty(Category, x, y),
'author': lambda x, y: _process_if_nonempty(Author, x, y),
'authors': lambda x, y: ([
Author(author, y)
for author in ensure_metadata_list(x)
] or _DISCARD),
'slug': lambda x, y: x.strip() or _DISCARD,
}
logger = logging.getLogger(__name__)
def ensure_metadata_list(text):
MARKDOWN = {
'extensions': [
'markdown.extensions.extra',
'markdown.extensions.footnotes',
'markdown.extensions.toc',
'markdown.extensions.meta'
],
'output_format': 'html5',
}
for dirpath, _, filenames in os.walk('/Users/rjames/Dropbox/blogs/ryanmoco/content/posts/Tech'):
for f in filenames:
if f.startswith('.'):
continue
filename, ext = os.path.splitext(f)
if ext != '.md':
continue
md = Markdown(**MARKDOWN)
with pelican_open(os.path.join(dirpath, f)) as text:
content = md.convert(text)
if hasattr(md, 'Meta'):
date = get_date(md.Meta['date'][0]).strftime('%Y-%m-%d')
# if not os.path.exists(os.path.join(dirpath, date)):
# os.mkdir(os.path.join(dirpath, date))
os.rename(
os.path.join(dirpath, f),
os.path.join(dirpath, date, f)
)
try:
from html import escape
except ImportError:
from cgi import escape
try:
from html.parser import HTMLParser
except ImportError:
from HTMLParser import HTMLParser
from pelican import signals
from pelican.contents import Page, Category, Tag, Author
from pelican.utils import get_date, pelican_open, FileStampDataCacher
METADATA_PROCESSORS = {
'tags': lambda x, y: [Tag(tag, y) for tag in x.split(',')],
'date': lambda x, y: get_date(x),
'modified': lambda x, y: get_date(x),
'status': lambda x, y: x.strip(),
'category': Category,
'author': Author,
'authors':
lambda x, y: [Author(author.strip(), y) for author in x.split(',')],
}
logger = logging.getLogger(__name__)
class BaseReader(object):
"""Base class to read files.
This class is used to process static files, and it can be inherited for
def updated_date(generator, metadata):
if 'updated' in metadata:
metadata['updated_date'] = get_date(metadata['updated'])
metadata['locale_updated_date'] = pelican_format_date(
metadata['updated_date'], metadata, generator.settings
)
