def list(self, posts):
self.__logger.debug('list - posts: {0}'.format(posts))
settings = Content().load_data_settings_yaml(self.__base_dir)
self.__logger.debug('list - settings: {0}'.format(settings))
# Starting with a dictionary as this is the easiest to find existing tags
tags = {}
for entry in posts:
directory = entry['directory']
file = entry['file']
meta, _, _ = Content().read_content(directory, file) # No need to catch the content
if meta['tags'] is None:
continue
for tag in meta['tags']:
label = TagsSupport().tag_label(tag)
if label in settings['skip_tags']:
self.__logger.debug('list - tag {0} found in skip_tags'.format(tag))
continue
if label in tags.keys():
self.__logger.debug('list - tag {0} already exists, +1'.format(tag))
current_count = tags[label]['count']
tags[label]['count'] = current_count + 1
else:
self.__logger.debug('list - tag {0} does not already exist'.format(tag))
data = {'label': label, 'count': 1, 'text': string.capwords(tag)}
tags[label] = data
self.__logger.debug('list - tags: '.format(tags))
# Pushing this into a simple array for Jinja2
tags_array = []
for _, value in tags.items(): # Only need the value
tags_array.append(value)
for tag in tags_array:
label = tag['label']
unmapped = '/tags/{0}'.format(label)
remapped = Remapper().remap_document(unmapped)
if unmapped != remapped:
label = remapped.split('/')[2]
tag['label'] = label
tag['text'] = TagsSupport().tag_text(label)
tags_list = sorted(tags_array, key=itemgetter('count'), reverse=True)
self.__logger.debug('list - sorted tags: '.format(tags_list))
return tags_list
def data(self, page, skip_tags):
self.__logger.debug('data - page: {0}'.format(page))
data = {}
meta, content, html = Content().read_content(self.__base_dir, '{0}.md'.format(page))
meta['content'] = html
data['page'] = meta
# remove the skipped tags
tags = []
try:
for tag in data['page']['tags']:
if TagsSupport().tag_label(tag) not in skip_tags:
tags.append(TagsSupport().tag_text(tag))
else:
self.__logger.debug('data - removing skipped tag: {0}'.format(tag))
except TypeError:
pass
except KeyError:
pass
data['page']['tags'] = tags
self.__logger.debug('data - tags: {0}'.format(tags))
data['page']['date'] = DateTimeSupport().rewrite_date(data['page']['date'])
self.__logger.debug('data - pages[{0}]: {1}'.format(page, data))
# meta: the meta in yaml format
# content: the content in pure markdown format
# data: the meta data injected with the html, completely ready for the post display
return meta, content, data
def __data(self):
content = Content().load_data_settings_yaml(self.__base_dir)
self.__logger.debug('__data - content: {0}'.format(content))
incoming = {}
outgoing = {}
for mapping in content['content']:
self.__logger.debug('__data - mapping: {0}'.format(mapping))
target = mapping['target']
source = mapping['source']
if target[0:1] != '/':
target = '/{0}'.format(target)
if source[0:1] != '/':
source = '/{0}'.format(source)
self.__logger.debug('__data - target: {0}'.format(target))
self.__logger.debug('__data - source: {0}'.format(source))
incoming[target] = source
self.__logger.info(
'mapping incoming URL \'{0}\' to source \'{1}\''.format(
target, source))
outgoing[source] = target
self.__logger.info(
'mapping outgoing source \'{0}\' to URL \'{1}\''.format(
source, target))
self.incoming = incoming
self.outgoing = outgoing
def data(self):
settings_dir = os.path.join(Options().data_dir, self.__base_dir)
self.__logger.debug('data - settings_dir: {0}'.format(settings_dir))
content = Content().load_yaml(settings_dir, 'global.yml')
self.__logger.debug('data - content: {0}'.format(content))
return content
def __get_settings(self):
if self.__settings is None:
content = Content().load_data_settings_yaml(self.__base_dir)
self.__logger.debug(
'__get_settings - content: {0}'.format(content))
self.__settings = content
return self.__settings
def skip_tags(self):
settings = Content().load_data_settings_yaml(self.__base_dir)
self.__logger.debug('list - settings: {0}'.format(settings))
tags = []
for tag in settings['skip_tags']:
tags.append(TagsSupport().tag_label(tag))
return tags
def data(self):
data = {}
i8n_dir = os.path.join(Options().data_dir, self.__base_dir)
for file in os.scandir(i8n_dir):
stem = Path(file).stem
self.__logger.debug('data - reading file {0}.yml'.format(stem))
data[stem] = Content().load_data_yaml(self.__base_dir, file)
self.__logger.debug('data - {0}'.format(data))
return data
def files_published(self):
directory = self.directory
files = []
for entry in self.files:
file = entry['file']
meta, _, _ = Content().read_content(directory, file) # only need the meta data
try:
value = meta['draft']
draft = bool(value)
except KeyError:
draft = False
self.__logger.debug('published_files - {0}/{1} is a draft: {2}'.format(directory, file, draft))
if not draft:
files.append(entry)
return files
def count_posts(self, posts, tag):
self.__logger.debug('count_posts - tag: {0}'.format(tag))
self.__logger.debug('count_posts - posts: {0}'.format(posts))
count_entries = 0
for entry in posts:
directory = entry['directory']
file = entry['file']
post, _, _ = Content().read_content(directory, file)
if post['tags'] is None:
continue
for tag_raw in post['tags']:
if TagsSupport().tag_label(tag_raw) == tag:
count_entries += 1
self.__logger.debug('count_posts - file {0} includes tag '.format(file))
break
self.__logger.debug('count_posts - tag {0} has {1} posts'.format(tag, count_entries))
return count_entries
def data(self):
content = Content().load_data_settings_yaml(self.__base_dir)
self.__logger.debug('data - content: {0}'.format(content))
return content
Options().environment = environment
Options().data_dir = data_dir
settings = SettingsLoader(environment).parse()
LoggingLoader().configure()
logger = logging.getLogger('HISTORY')
output = ''
for file in sorted(os.listdir(os.path.join(os.getcwd(), 'src', 'data', 'posts')), reverse=True):
logger.info('Parsing {0}'.format(file))
meta, content, _ = Content().read_content('posts', file)
if meta['tags'] is None:
continue
history = False
history_tags = ['history', 'History']
for ht in history_tags:
if ht in meta['tags']:
history = True
if not history:
continue
output += '# {0}\n'.format((meta['title']))
def data(self, posts, tag, page_index, index_max_posts, count_tag_posts):
self.__logger.debug('data - posts: {0}'.format(posts))
self.__logger.debug('data - tag: {0}'.format(tag))
self.__logger.debug('data - page_index tags: {0}'.format(page_index))
self.__logger.debug('data - index_max_posts tags: {0}'.format(index_max_posts))
self.__logger.debug('data - count_tag_posts tags: {0}'.format(count_tag_posts))
data = {'posts': []}
count_entries = 0
max_entries = index_max_posts
skip_entries = (int(page_index) - 1) * max_entries
self.__logger.debug('data - max_entries: {0}'.format(max_entries))
self.__logger.debug('data - skip_entries: {0}'.format(skip_entries))
for entry in posts:
directory = entry['directory']
file = entry['file']
post, _, post['content'] = Content().read_content(directory, file)
if post['tags'] is None:
continue
self.__logger.debug('data - post: {0}'.format(post))
must_include = False
for tag_raw in post['tags']:
if TagsSupport().tag_label(tag_raw) == tag:
must_include = True
break
self.__logger.debug('data - must_include: {0}'.format(must_include))
if must_include:
count_entries += 1
# We count the entries, but for pages 2 and more, you don't show them
if skip_entries >= count_entries:
self.__logger.debug('data - post skipped}')
continue
else:
self.__logger.debug('data - post added')
stem = Path(file).stem
url = '/posts/{0}'.format(stem)
url = Remapper().remap_document(url)
post['url'] = url
post['date'] = DateTimeSupport().rewrite_date(post['date'])
data['posts'].append(post)
if count_entries == (max_entries + skip_entries):
self.__logger.debug('data - enough posts')
break
data['tag'] = {'name': TagsSupport().tag_text(tag), 'path': tag}
total_index_pages = math.ceil(count_tag_posts / max_entries)
self.__logger.debug('data - total_index_pages: {0}'.format(total_index_pages))
data['pagination'] = {'current_page': int(page_index), 'total_pages': total_index_pages}
self.__logger.debug('data - {0}'.format(data))
return data
def data(self, query, page_index, search_base, max_entries):
self.__logger.debug('data - query: {0}'.format(query))
self.__logger.debug('data - page_index: {0}'.format(page_index))
self.__logger.debug('data - search_base: {0}'.format(search_base))
settings = Content().load_data_settings_yaml(self.__base_dir)
excluded = settings['excluded']
self.__logger.debug('data - excluded: {0}'.format(excluded))
data = {'found': []}
page_index = int(page_index)
count_entries = 0
skip_entries = (page_index - 1) * max_entries
first_entry = skip_entries + 1
for item in search_base:
self.__logger.debug('data - search_base item: {0}'.format(item))
directory = item['directory']
file = item['file']
stem = Path(file).stem
unmapped = '/{0}/{1}'.format(directory, stem)
remapped = Remapper().remap_document(unmapped)
if unmapped != remapped:
stem = remapped.split('/')[2]
# adding the stem to the object for the final url
item['stem'] = stem
if remapped in excluded:
self.__logger.debug('data - excluded item: {0}'.format(item))
continue
cached_already = DataCacher().cached_already(unmapped)
if cached_already:
self.__logger.debug('data - cached: {0}'.format(unmapped))
meta = DataCacher().get_cached('{0}/meta'.format(unmapped))
content = DataCacher().get_cached('{0}/content'.format(unmapped))
else:
self.__logger.debug('data - not cached: {0}'.format(unmapped))
meta, content, _ = Content().read_content(directory, file)
lowered_query = query.lower()
lowered_raw = content.lower()
self.__logger.debug('data - lowered_query: {0}'.format(lowered_query))
self.__logger.debug('data - lowered_raw: {0}'.format(lowered_raw))
if lowered_query in lowered_raw:
count_entries += 1
len_query = len(lowered_query)
if skip_entries >= count_entries:
self.__logger.debug('data - item skipped}')
continue
else:
self.__logger.debug('data - item added')
index = lowered_raw.find(lowered_query, 0)
extra = 70
prefix = ''
postfix = ''
start = 0
if index > extra:
self.__logger.debug('data - cutting text at the beginning')
prefix = '... '
start = index - extra
if index + extra < len(content):
self.__logger.debug('data - cutting text at the end')
postfix = " ..."
stop = start + (extra * 2) + len(query)
self.__logger.debug('data - index: {0}'.format(index))
self.__logger.debug('data - start: {0}'.format(start))
self.__logger.debug('data - stop: {0}'.format(stop))
# removing stuff until first blank
for c in range(start, index):
if content[c:c + 1] == ' ':
self.__logger.debug('data - blank found: {0}'.format(c))
start = c
break
# removing stuff until first blank, backwards
for c in range(stop, index + len(query), -1):
if content[c:c + 1] == ' ':
self.__logger.debug('data - blank found: {0}'.format(c))
stop = c
break
self.__logger.debug('data - spaced start: {0}'.format(start))
self.__logger.debug('data - spaced stop: {0}'.format(stop))
# vindevoy - 2020-05-06 - issue-187
# Do not use replace on sample here because replace is case-sensitive
sample = prefix + content[start:index] + '<b>' + content[index:index + len_query] + '</b>'
sample += content[index + len_query:stop] + postfix
self.__logger.debug('data - sample: {0}'.format(sample))
occurrences = lowered_raw.count(lowered_query)
self.__logger.debug('data - occurrences: {0}'.format(occurrences))
if not cached_already:
meta['date'] = DateTimeSupport().rewrite_date(meta['date'])
# cached data has the correct format already
entry = {
'item': item,
'meta': meta,
'occurrences': occurrences,
'sample': sample
}
data['found'].append(entry)
if count_entries == (max_entries + skip_entries):
self.__logger.debug('data - enough posts')
break
data['search'] = {'query': query}
last_entry = count_entries
data['pagination'] = {
'current_page': page_index,
'found_entries': count_entries - skip_entries,
'max_entries': max_entries,
'first_entry': first_entry,
'last_entry': last_entry
}
return data
def data(self, page_index, published_posts):
self.__logger.debug('data - page_index: {0}'.format(page_index))
data = {}
self.__logger.debug('data - common_data: {0}'.format(data))
data_index = {}
intro_meta, intro_content, data_index['introduction'] = Content(
).read_content(self.__base_dir, 'introduction.md')
data_index['image'] = intro_meta['image']
data['index'] = data_index
self.__logger.debug('data - data[index]: {0}'.format(data_index))
data['posts'] = []
data['spotlight_posts'] = []
data['highlight_posts'] = []
# For now we take the list of files in reversed sort order
# (newest should get a newer entry at the end of the list, like post1, ..., postx)
# I will sort this out later
max_entries = self.max_posts
spotlight_entries = self.spotlight_posts
highlight_entries = self.highlight_posts
self.__logger.debug('data - max_entries: {0}'.format(max_entries))
self.__logger.debug(
'data - spotlight_entries: {0}'.format(spotlight_entries))
self.__logger.debug(
'data - highlight_entries: {0}'.format(highlight_entries))
count_entries = 0
skip_entries = (int(page_index) - 1) * max_entries
self.__logger.debug('data - skip_entries: {0}'.format(skip_entries))
for entry in published_posts:
directory = entry['directory']
file = entry['file']
count_entries += 1
# We count the entries, but for pages 2 and more, you don't show them
if skip_entries >= count_entries:
continue
post, _, post['content'] = Content().read_content(directory, file)
stem = Path(file).stem
url = '/posts/{0}'.format(stem)
url = Remapper().remap_document(url)
post['url'] = url
post['date'] = DateTimeSupport().rewrite_date(post['date'])
self.__logger.debug('data - post: {0}'.format(post))
if page_index == 1:
if count_entries <= spotlight_entries:
self.__logger.debug(
'data - post added to spotlight_posts.')
data['spotlight_posts'].append(post)
if spotlight_entries < count_entries <= (spotlight_entries +
highlight_entries):
self.__logger.debug(
'data - post added to highlight_posts.')
data['highlight_posts'].append(post)
if count_entries > (spotlight_entries + highlight_entries):
self.__logger.debug(
'data - post added to (standard) posts.')
data['posts'].append(post)
else:
self.__logger.debug('data - post added to (standard) posts.')
data['posts'].append(post)
if count_entries == (max_entries + skip_entries):
self.__logger.debug('data - enough posts for this index page.')
break
total_posts = len(published_posts)
total_index_pages = math.ceil(total_posts / max_entries)
self.__logger.debug('data - total_posts: {0}'.format(total_posts))
self.__logger.debug(
'data - total_index_pages: {0}'.format(total_index_pages))
data['pagination'] = {
'current_page': int(page_index),
'total_pages': total_index_pages,
'spotlight_posts': len(data['spotlight_posts']),
'highlight_posts': len(data['highlight_posts']),
'posts': len(data['posts'])
}
self.__logger.debug('data - {0}'.format(data))
return data, intro_content
def footer_menu(self):
content = Content().load_data_settings_yaml(
self.__footer_menu_settings_dir)
self.__logger.debug('footer_menu: {0}'.format(content))
return content
}
}
for entry in DataLoader().posts_published:
file = entry['file']
stem = Path(file).stem
unmapped = '/posts/{0}'.format(stem)
remapped = Remapper().remap_document(unmapped)
logger.debug('unmapped: {0}'.format(unmapped))
logger.debug('remapped: {0}'.format(remapped))
logger.info('Parsing {0}'.format(file))
meta, _, _ = Content().read_content('posts', file)
xml += ' <url>\n'
xml += ' <loc>https://cherryblog.org{0}</loc>\n'.format(remapped)
xml += ' <lastmod>{0}</lastmod>\n'.format(meta['date'])
xml += ' <changefreq>never</changefreq>\n'
xml += ' <priority>{0}</priority>\n'.format(round(priority, 2))
xml += ' </url>\n'
if priority > 0.5:
priority = round(priority - 0.1, 2)
for entry in DataLoader().pages_published:
file = entry['file']
stem = Path(file).stem