def pages(self, page, **_):
start = datetime.now()
request = '/pages/{0}'.format(page)
self.__logger.debug('pages - request: {0}'.format(request))
if Options().caching and PageCacher().cached_already(request):
minified = PageCacher().get_cached(request)
else:
remapped = Remapper().remap_url(request)
self.__logger.debug('pages - remapped: {0}'.format(remapped))
if request != remapped:
page = remapped.split('/')[2]
self.__logger.debug('pages - page: {0}'.format(page))
# page on the URL: http://www.yoursite.ext/pages/page
data = DataLoader().pages_data(page)
data['url'] = remapped
template = TemplateLoader().get_template('screen_page.html')
rendered = template.render(data=data)
minified = html_minify(rendered)
if Options().caching:
PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
return minified
def __init__(self):
self.__logger = logging.getLogger('VIEW.TEMPLATE_LOADER')
self.__logger.setLevel(Options().default_logging_level)
# Theme dir already logged in main
self.__environment = Environment(
loader=FileSystemLoader(Options().theme_dir))
def tags(self, tag, page_index=1, **_):
start = datetime.now()
request = '/tags/{0}/{1}'.format(tag, page_index)
self.__logger.debug('tags - tag: {0}'.format(tag))
if Options().caching and PageCacher().cached_already(request):
minified = PageCacher().get_cached(request)
else:
short_request = 'tags/{0}'.format(tag)
self.__logger.debug('tags - short_request: {0}'.format(short_request))
remapped = Remapper().remap_url(short_request)
self.__logger.debug('tags - remapped: {0}'.format(remapped))
if short_request != remapped:
tag = remapped.split('/')[2]
self.__logger.debug('tags - tag: {0}'.format(tag))
data = DataLoader().tags_data(tag, page_index)
data['url'] = remapped
template = TemplateLoader().get_template('screen_tag.html')
rendered = template.render(data=data)
minified = html_minify(rendered)
if Options().caching:
PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
return minified
def posts(self, post, **_):
start = datetime.now()
request = '/posts/{0}'.format(post)
self.__logger.debug('posts - request: {0}'.format(request))
if Options().caching and PageCacher().cached_already(request):
minified = PageCacher().get_cached(request)
else:
remapped = Remapper().remap_url(request)
self.__logger.debug('posts - remapped: {0}'.format(remapped))
if request != remapped:
post = remapped.split('/')[2]
self.__logger.debug('posts - post: {0}'.format(post))
data = DataLoader().posts_data(post)
data['url'] = remapped
template = TemplateLoader().get_template('screen_post.html')
rendered = template.render(data=data)
minified = html_minify(rendered)
if Options().caching:
PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
return minified
def __get_data(key, cls, method):
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
data = getattr(cls, method)
if Options().caching:
DataCacher().cache(key, data)
return data
def tags_skip_list(self):
key = 'tags_skip_list'
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
tags_list = Tags().skip_tags()
if Options().caching:
DataCacher().cache(key, tags_list)
return tags_list
def configure():
cherrypy.engine.unsubscribe('graceful', cherrypy.log.reopen_files)
# DO NOT USE Content() here, it's not ready
settings_file = os.path.join(Options().data_dir, 'logging',
'settings.yml')
file = open(settings_file, 'r')
settings_yaml = yaml.load(file.read(), Loader=yaml.SafeLoader)
logging.config.dictConfig(settings_yaml)
Options().default_logging_level = settings_yaml['loggers']['']['level']
def load_data_yaml(self, directory, file):
self.__logger.debug(
'load_data_yaml - Loading {0} from directory {1}.'.format(
file, directory))
return self.load_yaml(os.path.join(Options().data_dir, directory),
file)
def __split_file(self, data):
if self.__meta_content_separator is None:
self.__meta_content_separator = Options().meta_content_separator
self.__logger.debug(
'__split_file - Split file separator is {0}'.format(
self.__meta_content_separator))
split = data.split(self.__meta_content_separator)
meta = split[0]
content_raw = ""
if len(split) == 2:
content_raw = split[1]
else:
self.__logger.debug('__split_file - No content found.')
meta_data = yaml.load(meta, Loader=yaml.SafeLoader)
self.__logger.debug('__split_file - Meta data:\n{0}'.format(meta_data))
self.__logger.debug(
'__split_file - Markdown data:\n{0}'.format(content_raw))
content_html = markdown.markdown(content_raw)
self.__logger.debug('__split_file - HTML:\n{0}'.format(content_html))
return meta_data, content_raw, content_html
def load_data_settings_yaml(self, directory):
self.__logger.debug(
'load_data_settings_yaml - '
'Loading settings.yml from data sub-directory {0}.'.format(
directory))
return self.load_settings_yaml(
os.path.join(Options().data_dir, directory))
def tags_posts_count(self, tag):
key = 'tags_posts_count/{0}'.format(tag)
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
if Options().include_drafts:
posts = self.posts_files
else:
posts = self.posts_published
data = Tags().count_posts(posts, tag)
if Options().caching:
DataCacher().cache(key, data)
return data
def posts_data(self, post):
key = '/posts/{0}'.format(post)
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
common = self.common_data
meta, content, data = Posts().data(post, self.tags_skip_list)
combined = self.__combine(common, data)
if Options().caching:
DataCacher().cache(key, combined)
DataCacher().cache('{0}/meta'.format(key), meta)
DataCacher().cache('{0}/content'.format(key), content)
return combined
def pages_data(self, page):
key = '/pages/{0}'.format(page)
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
common = self.common_data
meta, content, data = Pages().data(
page, self.tags_skip_list) # No catching the meta and raw data yet
combined = self.__combine(common, data)
if Options().caching:
DataCacher().cache(key, combined)
DataCacher().cache('{0}/meta'.format(key), meta)
DataCacher().cache('{0}/content'.format(key), content)
return combined
def tags_list(self):
key = 'tags_list'
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
if Options().include_drafts:
posts = self.posts_files
else:
posts = self.posts_published
tags_list = Tags().list(posts)
if Options().caching:
DataCacher().cache(key, tags_list)
return tags_list
def data(self):
settings_dir = os.path.join(Options().data_dir, self.__base_dir)
self.__logger.debug('data - settings_dir: {0}'.format(settings_dir))
content = Content().load_yaml(settings_dir, 'global.yml')
self.__logger.debug('data - content: {0}'.format(content))
return content
def tags_data(self, tag, page_index):
key = 'tags_data/{0}/{1}'.format(tag, page_index)
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
if Options().include_drafts:
posts = self.posts_files
else:
posts = self.posts_published
common = self.common_data
data = Tags().data(posts, tag, page_index, self.index_max_posts,
self.tags_posts_count(tag))
combined = self.__combine(common, data)
if Options().caching:
DataCacher().cache(key, combined)
return combined
def index_data(self, page_index):
key = '/index/{0}'.format(page_index)
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
common = self.common_data
if Options().include_drafts:
posts = self.posts_files
else:
posts = self.posts_published
data, _ = Index().data(page_index, posts)
# We don't care yet about the introduction content
combined = self.__combine(common, data)
if Options().caching:
DataCacher().cache(key, combined)
return combined
def common_data(self):
key = 'common_data'
if Options().caching and DataCacher().cached_already(key):
return DataCacher().get_cached(key)
data = {
'i8n': self.i8n,
'settings': self.global_settings,
'tags_list': self.tags_list,
'tags_list_count': self.tags_list_count,
'tags_skip_list': self.tags_skip_list,
'main_menu': self.index_main_menu,
'footer_menu': self.index_footer_menu,
'important_news': self.important_news_data,
'code_version': self.code_version_data
}
if Options().caching:
DataCacher().cache(key, data)
return data
def index(self, page_index=1, **_):
start = datetime.now()
request = '/index/{0}'.format(page_index)
if Options().caching and PageCacher().cached_already(request):
minified = PageCacher().get_cached(request)
else:
data = DataLoader().index_data(page_index)
data['url'] = request
template = TemplateLoader().get_template('screen_index.html')
rendered = template.render(data=data)
minified = html_minify(rendered)
if Options().caching:
PageCacher().cache(request, minified)
finished = datetime.now()
self.__logger.info('{0} {1}'.format(request, finished - start))
return minified
def data(self):
data = {}
i8n_dir = os.path.join(Options().data_dir, self.__base_dir)
for file in os.scandir(i8n_dir):
stem = Path(file).stem
self.__logger.debug('data - reading file {0}.yml'.format(stem))
data[stem] = Content().load_data_yaml(self.__base_dir, file)
self.__logger.debug('data - {0}'.format(data))
return data
def read_content(self, directory, file):
content_dir = os.path.join(Options().data_dir, directory)
self.__logger.debug(
'read_content - Reading content file {0} from directory {0}.'.
format(file, content_dir))
try:
content_file = open(os.path.join(content_dir, file), 'r')
except FileNotFoundError:
self.__logger.warning(
'COULD NOT FIND THE YAML FILE {0}/{1}'.format(directory, file))
return {}, ''
meta, content, html = self.__split_file(content_file.read())
# No logging, already logged
return meta, content, html
def parse(self):
# data_dir and environment are set before the SettingsLoader is called
# and are read in application.py where the command line is parsed
environment_dir = os.path.join(Options().data_dir, 'environment')
# read the yaml file
# DO NOT USE the Content() class here, it needs the settings itself to set the logging level
settings_file = os.path.join(environment_dir,
'{0}.yml'.format(self.__environment))
file = open(settings_file, 'r')
settings_yaml = yaml.load(file.read(), Loader=yaml.SafeLoader)
# set the settings needed elsewhere in the code
self.__option_settings(settings_yaml)
# Return the settings really needed in a format for CherryPy
# They will be used when starting the engine in application.py
# They are logged in main.py
return self.__engine_settings(settings_yaml)
def search_data(self, query, page_index):
search_base = []
if Options().include_drafts:
pages = self.pages_files
posts = self.posts_files
else:
pages = self.pages_published
posts = self.posts_published
for page in pages:
search_base.append(page)
for post in posts:
search_base.append(post)
common = self.common_data
data = Search().data(query, page_index, search_base,
self.index_max_posts)
combined = self.__combine(common, data)
# search data is not stored in memory because it could potentially eat all the memory
return combined
def __init__(self):
self.__logger = logging.getLogger('MODEL.IMPORTANT_NEWS')
self.__logger.setLevel(Options().default_logging_level)
#!/bin/python
if __name__ == '__main__':
import json
from common.options import Options
config = Options()
config.ENGINE = 'django.db.backends.mysql'
config.HOST = 'host'
config.PORT = 'port'
config.NAME = 'database'
config.USER = '******'
config.PASSWORD = '******'
config.OPTIONS.charset = 'utf8mb4'
config.OPTIONS.init_command = "SET sql_mode='STRICT_TRANS_TABLES'"
print(json.dumps(config))
# - Builds the history page
#
###
import logging
import os
from common.options import Options
from common.content import Content
from controller.logging_loader import LoggingLoader
from controller.settings_loader import SettingsLoader
environment = 'localhost'
data_dir = os.path.join(os.getcwd(), 'src', 'data')
Options().environment = environment
Options().data_dir = data_dir
settings = SettingsLoader(environment).parse()
LoggingLoader().configure()
logger = logging.getLogger('HISTORY')
output = ''
for file in sorted(os.listdir(os.path.join(os.getcwd(), 'src', 'data', 'posts')), reverse=True):
logger.info('Parsing {0}'.format(file))
meta, content, _ = Content().read_content('posts', file)
def __init__(self):
self.__logger = logging.getLogger('MODEL.CODE_VERSION')
self.__logger.setLevel(Options().default_logging_level)
def __init__(self):
self.__logger = logging.getLogger('REMAPPER')
self.__logger.setLevel(Options().default_logging_level)
environment = 'localhost'
data_dir = ""
opts, args = getopt.getopt(sys.argv[1:], 'd:e:', ['env=', 'data='])
for opt, arg in opts:
if opt in ['-d', '--data']:
data_dir = arg
if opt in ['-e', '--env']:
environment = arg
if data_dir == '':
data_dir = os.path.join(os.getcwd(), 'src', 'data')
# Options is a singleton and can be loaded with what we know already
Options().environment = environment
Options().data_dir = data_dir
# Load the settings from the environment.yml file to fill out the rest of the settings
# This also sets most of the unknown properties in Options()
settings = SettingsLoader(environment).parse()
# Do not load the logging before you have the data_dir
# This will fill out the logging level in the Options()
LoggingLoader().configure()
logger = logging.getLogger('MAIN')
logger.info('{0} v.{1}'.format(__application, __version))
logger.info('Environment set to {0}.'.format(environment))
logger.info('Data directory set to {0}.'.format(data_dir))
def __init__(self):
self.__logger = logging.getLogger('COMMON.CONTENT')
self.__logger.setLevel(Options().default_logging_level)