class ServerDataWriter:
def __init__(self, *args, **kwargs):
self.tplccdir = kwargs['tplccdir'] if 'tplccdir' in kwargs else None
self.tpldirs = kwargs['tpldirs'] if 'tpldirs' in kwargs else [os.curdir]
if 'tpls' not in kwargs or not isinstance(kwargs['tpls'], dict):
exit_with_error("Output templates not defined")
tpls_dict = {a: kwargs['tpls'][a] if a in kwargs['tpls'] and \
isinstance(kwargs['tpls'][a], dict) else {} \
for a in ['files', 'parmap']}
self.tpls = type(
self.__class__.__name__ + \
".Templates",
(object,),
tpls_dict
)
tplookup_kwargs = {
"directories": self.tpldirs,
"output_encoding": 'utf-8',
"encoding_errors": 'replace',
"strict_undefined": True
}
if self.tplccdir:
tplookup_kwargs["module_directory"] = self.tplccdir
self.tplookup = TemplateLookup(**tplookup_kwargs)
def render_tpl(self, tpl):
if tpl not in self.tpls.files:
exit_with_error("Template file not specified for template %s" % tpl)
elif not self.tplookup.has_template(self.tpls.files[tpl]):
exit_with_error("Template file not found: %s" % self.tpls.files[tpl])
t = self.tplookup.get_template(self.tpls.files[tpl])
return t.render(**self.tpls.parmap[tpl])
class Mailer:
def __init__(self, target_address: str):
self.target_address = target_address
self.template_lookup = TemplateLookup(
directories=[
os.path.join(os.path.dirname(__file__), 'mail_templates')
],
strict_undefined=True,
)
def connect(self) -> smtplib.SMTP:
return smtplib.SMTP(self.target_address, 25)
def async_mailer(self) -> aiosmtplib.SMTP:
return aiosmtplib.SMTP(self.target_address, 25)
def _render_template(self, language: str, name: str,
**kwargs) -> Tuple[str, str]:
if language != 'en_us' and not self.template_lookup.has_template(
f'{language}/{name}'):
language = 'en_us'
template = self.template_lookup.get_template(f'{language}/{name}')
data = template.render(
config=config,
**kwargs,
)
return data.split('\n', 1)
async def async_send_mail(self, language: str, name: str, from_addr: str,
to: List[str], context: dict):
html_title, html_data = self._render_template(language, name + '.html',
**context)
txt_title, txt_data = self._render_template(language, name + '.txt',
**context)
assert txt_title == html_title
message = MIMEMultipart('alternative')
message['Subject'] = txt_title
message.attach(MIMEText(html_data, 'html'))
message.attach(MIMEText(txt_data, 'plain'))
async with self.async_mailer() as connected_mailer:
await connected_mailer.sendmail(from_addr, to, message.as_bytes())
async def async_send_mail_raw(self, from_addr: str, to: List[str],
content: bytes):
async with self.async_mailer() as connected_mailer:
await connected_mailer.sendmail(from_addr, to, content)
class ServerDataWriter:
def __init__(self, *args, **kwargs):
self.tplccdir = kwargs['tplccdir'] if 'tplccdir' in kwargs else None
self.tpldirs = kwargs['tpldirs'] if 'tpldirs' in kwargs else [
os.curdir
]
if 'tpls' not in kwargs or not isinstance(kwargs['tpls'], dict):
exit_with_error("Output templates not defined")
tpls_dict = {a: kwargs['tpls'][a] if a in kwargs['tpls'] and \
isinstance(kwargs['tpls'][a], dict) else {} \
for a in ['files', 'parmap']}
self.tpls = type(
self.__class__.__name__ + \
".Templates",
(object,),
tpls_dict
)
tplookup_kwargs = {
"directories": self.tpldirs,
"output_encoding": 'utf-8',
"encoding_errors": 'replace',
"strict_undefined": True
}
if self.tplccdir:
tplookup_kwargs["module_directory"] = self.tplccdir
self.tplookup = TemplateLookup(**tplookup_kwargs)
def render_tpl(self, tpl):
if tpl not in self.tpls.files:
exit_with_error("Template file not specified for template %s" %
tpl)
elif not self.tplookup.has_template(self.tpls.files[tpl]):
exit_with_error("Template file not found: %s" %
self.tpls.files[tpl])
t = self.tplookup.get_template(self.tpls.files[tpl])
return t.render(**self.tpls.parmap[tpl])
class MessageTemplate(object):
def __init__(self):
self.lookup = TemplateLookup(settings.TEMPLATE_DIRS)
def _get_template(self, event):
filename = event.replace(' ', '-').lower()
template_name = "/{0}.tpl".format(filename)
if self.lookup.has_template(template_name):
return self.lookup.get_template(template_name)
def get_message(self, event, json):
template = self._get_template(event)
if template is None:
return
try:
message = template.render(**json)
except (KeyError, TypeError, IndexError):
logger.exception("Template Render Failed.")
return
# use replace so. Carriage returns not allowed in privmsg(text)
# https://github.com/jaraco/irc/blob/master/irc/client.py#L900
return message.replace('\n', '').replace('\r', '').strip()
class PyDecanter(object):
RE_CACHEBUSTABLE = re.compile(r"(.*)\.(jpe?g|gif|png|svg|ttf|woff|woff2)")
RE_CACHEBUSTER = re.compile(
r"(.*)\.[0-9a-f]{12}\.(jpe?g|gif|png|svg|ttf|woff|woff2)")
CACHE_TAG_EXCS = (
"favicon",
"apple-touch-icon",
"android-chrome",
"mstile",
"safari-pinned-tab",
)
# A tuple of regular expressions representing files which should not be
# output as part of the build process.
PRIVATE_FILES = (
re.compile(r".*\.mako$"),
re.compile(r"(^|.*\/)\.(?!htaccess)"),
re.compile(r"^\.build"),
re.compile(r"^\.git"),
re.compile(r"^\.sass-cache"),
)
# A tuple of extensions for files that should be processed by Mako
TEMPLATE_EXTS = ("html", )
def __init__(self, args):
# the Bottle application
self.app = bottle.Bottle()
# the base root from which to serve files
# (mako needs a string, not a Path() object).
self.base_root = str(args.base_root.resolve())
self.base_url = re.sub(r"/+", "/", "/{0}/".format(args.base_url))
self.assets_dir = args.assets_dir
if "private" in args:
self.PRIVATE_FILES = self.PRIVATE_FILES + tuple(
re.compile(_) for _ in args.private.split(","))
# the TemplateLookup of Mako
self.templates = TemplateLookup(
directories=[self.base_root],
imports=[
"from markdown import markdown",
"from lib.typogrify import amp, caps, initial_quotes,"
" smartypants, titlecase, typogrify, widont",
"from lib.decorators import css, js",
],
input_encoding="UTF8",
collection_size=100,
default_filters=["trim"],
)
self.context = {
"base_url": self.base_url.rstrip("/"),
"base_root": Path(self.base_root),
"assets_dir": self.assets_dir,
"output_root": self.base_root,
"compress": False,
}
# establish routing config for bottle
self.app.route("{0}<path:path>".format(self.base_url),
method="GET",
callback=self.render)
self.app.route(self.base_url,
method="GET",
callback=lambda: self.render("index.html"))
# if there's a 404.html file in the root folder, render this for
# 404 responses (this returns the wrong response code, of course,
# but there's no point in modifying this on a dev-only server).
if os.path.isfile(os.path.join(self.base_root, "404.html")):
self.app.error(404)(lambda err: self.render("404.html"))
# prep the simple WSGI server (could use waitress here, but the extra
# dependency is not really necessary for dev-only purposes).
self.server = make_server(args.host, args.port, self,
ThreadingWSGIServer)
def restart(modified_path, base_dir=os.getcwd()):
"""automatically restart the server if changes are made to python
files on this path."""
self.server.server_close()
self.server.shutdown()
logging.info("change detected to '%s' -- restarting server",
modified_path)
args = sys.argv[:]
args.insert(0, sys.executable)
os.chdir(base_dir)
os.execv(sys.executable, args)
monitor = Monitor(interval=1.0)
if args.ini_file is not None:
monitor.track(
str(Path(os.path.expanduser(args.ini_file)).resolve()))
monitor.on_modified = restart
def is_private(self, path):
"""returns True if the path represents a file which should not be
output as part of the static site, and False otherwise."""
for regex in self.PRIVATE_FILES:
if regex.match(path):
return True
return False
def render(self, path):
if os.path.isdir(os.path.join(self.base_root, path)):
path = os.path.join(path, "index.html")
if self.is_private(path) and not path.startswith(self.assets_dir):
return bottle.HTTPError(
403, "Direct access to {0} is forbidden".format(path))
if path.split(
".")[-1] in self.TEMPLATE_EXTS and self.templates.has_template(
path):
template = self.templates.get_template(path)
self.context.update({"path": path})
document = template.render_unicode(**self.context)
if self.context["compress"]:
document = self.add_image_cache_tags(document)
document = self.tidy_html(document)
return document.encode("utf-8")
# if using the static filters whilst using the dev. server, we need to
# remove the hash from image assets (note that the build_static
# function will generate image assets which actually _do_ have these
# hashes in the filename, so there is no need for a similar rewrite
# in the apache2/nginx config).
if self.RE_CACHEBUSTER.match(path):
path = self.RE_CACHEBUSTER.sub(r"\1.\2", path)
# bottle does not detext some mimetypes correctly, so here we check for
# any that need to be hard-coded.
for ext, mimetype in FORCE_MIMETYPES:
if path.endswith(ext):
return bottle.static_file(path,
root=self.base_root,
mimetype=mimetype)
return bottle.static_file(path, root=self.base_root)
@staticmethod
def tidy_html(document):
# pre-process to remove comments (htmltidy's option to do so will
# also strip IE conditional comments).
soup = BeautifulSoup(document, "lxml")
for cmt in soup.findAll(text=lambda text: isinstance(text, Comment)):
if not (cmt.startswith("[if") or cmt.startswith("<![endif")):
cmt.extract()
document, errors_ = tidy_document(str(soup), options=TIDY_OPTIONS)
# clean-up some line-breaks associated with conditional comments
document = re.sub(r"(?<!\s)<!--", "\n<!--", document)
# adjust some of the htmltidy indenting to my own taste :)
document = re.sub(
r"(?m)^(\s+)(.*?)(<script[^>]*>)\n</script>",
r"\1\2\n\1\3</script>",
document,
)
document = re.sub(
r"(?m)^(\s+)(.*?)(<script[^>]*>)\n((?:[^\n]+\n)+)\s+</script>",
r"\1\2\n\1\3\n\1\4\1</script>",
document,
)
re_single_line_elems = re.compile(
r"^(\s*<([a-z][a-z0-9]*)\b[^>]*>)\s*([^\n]*?)\s*</\2>",
re.DOTALL | re.MULTILINE,
)
document = re_single_line_elems.sub(r"\1\3</\2>", document)
return document
def add_image_cache_tags(self, document):
re_svgfallback = re.compile(
r"this.onerror=null;\s*this.src='([^\']+\.(?:png|gif|jpe?g))'")
soup = BeautifulSoup(document, "lxml")
for img in soup.findAll("img"):
if not any([exc in img["src"] for exc in self.CACHE_TAG_EXCS]):
img["src"] = add_cache_tag(img["src"], self.base_url,
self.base_root,
self.context["path"])
# also need to deal with attributes of the form:
# onerror="this.onerror=null; this.src='<fallback_image>'"
if "onerror" in img.attrs and re_svgfallback.match(img["onerror"]):
img["onerror"] = re_svgfallback.sub(
lambda m: m.group(0).replace(
m.group(1),
add_cache_tag(m.group(1), self.base_url, self.base_root
),
),
img["onerror"],
)
# deal with <a href="<cache-tagged image here>">
re_local_image = re.compile("^" + self.base_url +
r"[^/].+\.(?:png|gif|jpe?g)$")
for anchor in soup.findAll("a", href=re_local_image):
if not any([exc in anchor["href"] for exc in self.CACHE_TAG_EXCS]):
anchor["href"] = add_cache_tag(anchor["href"], self.base_url,
self.base_root,
self.context["path"])
return str(soup)
@staticmethod
def create_cache_dir(cache_dir, cleanup=True):
"""create a cache folder on the base_root path, and automatically
remove it again when the program exits (unless it was already
there to begin with).
"""
if not os.path.isdir(cache_dir):
try:
os.makedirs(cache_dir)
except OSError as exc: # Python >2.5
if exc.errno == errno.EEXIST:
pass
else:
raise
def cleanup_func():
""" Helper function to remove the temporary cache folder. """
shutil.rmtree(cache_dir)
if cleanup:
atexit.register(cleanup_func)
def run(self, args):
""" Launch a development web server. """
# apply any context passed in from a config file
self.context.update(args.context)
# if we're running with `static = True`, but running a dev. server,
# we need a (temporary) cache folder to be available.
if args.compress:
self.context.update({"compress": True})
self.assets_dir = os.path.join(self.base_root, self.assets_dir)
self.create_cache_dir(self.assets_dir)
try:
logging.info(
"server running at http://%s%s",
"{}:{}".format(*self.server.server_address),
self.base_url,
)
self.server.serve_forever(poll_interval=0.5)
except KeyboardInterrupt:
logging.info("Quitting Server!")
self.server.server_close()
self.server.shutdown()
raise SystemExit
def get(self, path):
""" get the content of a url as rendered by bottle """
handler = SimpleHandler(sys.stdin, sys.stdout, sys.stderr, {})
handler.setup_environ()
env = handler.environ
env.update({
"PATH_INFO": "{0}/{1}".format(self.base_url, path),
"REQUEST_METHOD": "GET",
})
out = b"".join(self.app(env, lambda *args: None))
return out
def build_static(self, args):
"""
Generates a complete static version of the web site. It will stored in
output_folder.
"""
# apply any context passed in from a config file
self.context.update(args.context)
self.assets_dir = os.path.join(args.output_dir, self.assets_dir)
self.create_cache_dir(self.assets_dir, False)
self.context.update({
"output_root": args.output_dir,
"compress": args.compress
})
public_files = []
for dirpath, dirnames_, filenames in os.walk(self.base_root):
for filename in filenames:
path = os.path.relpath(os.path.join(dirpath, filename),
self.base_root)
if not self.is_private(path):
public_files.append(path)
for filepath in public_files:
# if filepath represents a cacheable asset, add a (hash) tag
# to the output filename.
if (self.context["compress"]
and self.RE_CACHEBUSTABLE.match(filepath)
and not any(exc in filepath
for exc in self.CACHE_TAG_EXCS)):
filehash = get_file_hash(
os.path.join(self.base_root, filepath), 12)
output_path = self.RE_CACHEBUSTABLE.sub(
lambda m, fh=filehash: ".".join(
(m.group(1), fh, m.group(2))),
filepath,
)
output_path = os.path.join(args.output_dir, output_path)
else:
output_path = os.path.join(args.output_dir, filepath)
dirname = os.path.dirname(output_path)
if not os.path.exists(dirname):
os.makedirs(dirname)
if filepath.endswith((".html", ".css", ".js")):
logging.info(colored("generating %s", "blue"), filepath)
content = self.get(filepath)
with open(output_path, "wb") as _fh:
_fh.write(content)
else:
# just copy the file instead
logging.info(colored("copying %s", "green"), filepath)
shutil.copy2(os.path.join(self.base_root, filepath),
output_path)
def __call__(self, environ, start_response):
return self.app(environ, start_response)
class Renderer(object):
'''Manages rendering templates'''
def __init__(self, template_dir, invoking_filenames, source_files):
self.template_dir = template_dir
self.invoking_filenames = invoking_filenames
self.source_files = source_files
self.tl = TemplateLookup(directories=[template_dir])
self.template_context = {
'camel': camel, # CamelCase function
'dromedary': dromedary, # dromeDary case function
'EmptyTemplate': EmptyTemplate,
}
self.mtime_cache = {}
self.dependency_cache = {}
def mtime(self, filename):
try:
return self.mtime_cache[filename]
except KeyError:
mtime = os.path.getmtime(filename)
self.mtime_cache[filename] = mtime
return mtime
def render(self,
template_name,
output_dir,
output_name=None,
dependencies=None,
**kwargs):
if output_name is None:
output_name = template_name
tpl = self.tl.get_template(template_name)
output_path = output_dir + '/' + output_name
if self.already_rendered(tpl, output_path, dependencies):
logger.debug(" Up to date: %s", output_path)
return
results = self.template_context.copy()
results.update(kwargs)
try:
rendered = tpl.render(**results)
except EmptyTemplate:
logger.debug(" Empty template: %s", output_path)
return
with codecs.open(output_path, "w", "utf-8") as outfile:
logger.info("Rendering %s", output_path)
outfile.write(
self.autogenerated_header(
self.template_dir + '/' + template_name,
output_path,
self.invoking_filenames,
))
outfile.write(rendered)
def get_template_name(self, classname, directory, default):
'''Returns the template for this class'''
override_template = '{}/{}.java'.format(directory, classname)
if self.tl.has_template(override_template):
logger.debug("Found an override for %s at %s", classname,
override_template)
return override_template
else:
return default
def dependent_templates(self, tpl):
'''Returns filenames for all templates that are inherited from the
given template'''
if tpl.filename in self.dependency_cache:
return self.dependency_cache[tpl.filename]
inherit_files = re.findall(r'inherit file="(.*)"', tpl.source)
op = os.path
dependencies = set()
tpl_dir = op.dirname(tpl.filename)
for parent_relpath in inherit_files:
parent_filename = op.normpath(op.join(tpl_dir, parent_relpath))
dependencies.add(parent_filename)
dependencies.update(
self.dependent_templates(
self.tl.get_template(
self.tl.filename_to_uri(parent_filename))))
dependencies.add(tpl.filename)
self.dependency_cache[tpl.filename] = dependencies
return dependencies
def already_rendered(self, tpl, output_path, dependencies):
'''Check if rendered file is already up to date'''
if not os.path.exists(output_path):
logger.debug(" Rendering since %s doesn't exist", output_path)
return False
output_mtime = self.mtime(output_path)
# Check if this file or the invoking file have changed
for filename in self.invoking_filenames + self.source_files:
if self.mtime(filename) > output_mtime:
logger.debug(" Rendering since %s has changed", filename)
return False
if self.mtime(__file__) > output_mtime:
logger.debug(" Rendering since %s has changed", __file__)
return False
# Check if any dependent templates have changed
for tpl in self.dependent_templates(tpl):
if self.mtime(tpl) > output_mtime:
logger.debug(" Rendering since %s is newer", tpl)
return False
# Check if any explicitly defined dependencies have changed
dependencies = dependencies or []
for dep in dependencies:
if self.mtime(dep) > output_mtime:
logger.debug(" Rendering since %s is newer", dep)
return False
return True
def autogenerated_header(self, template_path, output_path, filename):
rel_tpl = os.path.relpath(template_path, start=output_path)
filenames = ' and '.join(
os.path.basename(f) for f in self.invoking_filenames)
return ('// Autogenerated by {}.\n'
'// Do not edit this file directly.\n'
'// The template for this file is located at:\n'
'// {}\n').format(filenames, rel_tpl)
class PyBlue:
TEMPLATE_DIR = op.abspath(op.join(op.split(__file__)[0], "templates"))
def __init__(self):
# the Bottle application
self.app = bottle.Bottle()
# a set of strings that identifies the extension of the files
# that should be processed using Mako
self.template_exts = set([".html"])
# the folder where the files to serve are located. Do not set
# directly, use set_folder() instead
self.folder = "."
# the list of all crawled files, initialized in set_folder()
self.files = []
# should it recrawl directories on each request, used in serving
# can be turned off in the options
self.refresh = False
# the TemplateLookup of Mako
self.templates = TemplateLookup(directories=[self.folder, self.TEMPLATE_DIR],
imports=[ "from pyblue.utils import markdown", "from pyblue.utils import rst, asc"],
input_encoding='iso-8859-1',
collection_size=100,
)
# A list of regular expression. Files whose the name match
# one of those regular expressions will not be outputed when generating
# a static version of the web site
self.file_exclusion = [r".*\.mako", r".*\.py", r"(^|.*\/)\..*"]
def is_public(path):
for ex in self.file_exclusion:
if re.match(ex, path):
return False
return True
def base_lister():
files = []
for dirpath, dirnames, filenames in os.walk(self.folder):
filenames.sort()
for f in filenames:
absp = os.path.join(dirpath, f)
path = os.path.relpath(absp, self.folder)
files.append(path)
files = filter(is_public, files)
return files
# A list of function. Each function must return a list of paths
# of files to export during the generation of the static web site.
# The default one simply returns all the files contained in the folder.
# It is necessary to define new listers when new routes are defined
# in the Bottle application, or the static site generation routine
# will not be able to detect the files to export.
self.file_listers = [base_lister]
def file_renderer(path):
if is_public(path):
_, ext = os.path.splitext(path)
if ext in self.template_exts and self.templates.has_template(path):
f = File(fname=path, root=self.folder)
t = self.templates.get_template(path)
if self.refresh:
self.files = self.collect_files()
try:
data = t.render_unicode(p=self, f=f, u=utils)
# if it made it this far the user wants these to be rendered as html
if ext == '.md':
data = utils.markdown(data)
elif ext == '.rst':
data = utils.rst(data)
page = data.encode(t.module._source_encoding)
return page
except Exception, exc:
_logger.error("error %s generating page %s" % (exc, path))
return exceptions.html_error_template().render()
return bottle.static_file(path, root=self.folder)
return bottle.HTTPError(404, 'File does not exist.')
# The default function used to render files. Could be modified to change the way files are
# generated, like using another template language or transforming css...
self.file_renderer = file_renderer
self.app.route('/', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda: self.file_renderer('index.html'))
self.app.route('/<path:path>', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda path: self.file_renderer(path))
class PyGreen:
def __init__(self):
# the Bottle application
self.app = flask.Flask(__name__, static_folder=None, template_folder=None)
# a set of strings that identifies the extension of the files
# that should be processed using Mako
self.template_exts = set(["html"])
# the folder where the files to serve are located. Do not set
# directly, use set_folder instead
self.folder = "."
self.app.root_path = "."
# the TemplateLookup of Mako
self.templates = TemplateLookup(directories=[self.folder],
imports=["from markdown import markdown"],
input_encoding='iso-8859-1',
collection_size=100,
)
# A list of regular expression. Files whose the name match
# one of those regular expressions will not be outputed when generating
# a static version of the web site
self.file_exclusion = [r".*\.mako", r".*\.py", r"(^|.*\/)\..*"]
def is_public(path):
for ex in self.file_exclusion:
if re.match(ex,path):
return False
return True
def base_lister():
files = []
for dirpath, dirnames, filenames in os.walk(self.folder):
for f in filenames:
absp = os.path.join(dirpath, f)
path = os.path.relpath(absp, self.folder)
if is_public(path):
files.append(path)
return files
# A list of functions. Each function must return a list of paths
# of files to export during the generation of the static web site.
# The default one simply returns all the files contained in the folder.
# It is necessary to define new listers when new routes are defined
# in the Bottle application, or the static site generation routine
# will not be able to detect the files to export.
self.file_listers = [base_lister]
def file_renderer(path):
if is_public(path):
if path.split(".")[-1] in self.template_exts and self.templates.has_template(path):
t = self.templates.get_template(path)
data = t.render_unicode(pygreen=self)
return data.encode(t.module._source_encoding)
if os.path.exists(os.path.join(self.folder, path)):
return flask.send_file(path)
flask.abort(404)
# The default function used to render files. Could be modified to change the way files are
# generated, like using another template language or transforming css...
self.file_renderer = file_renderer
self.app.add_url_rule('/', "root", lambda: self.file_renderer('index.html'), methods=['GET', 'POST', 'PUT', 'DELETE'])
self.app.add_url_rule('/<path:path>', "all_files", lambda path: self.file_renderer(path), methods=['GET', 'POST', 'PUT', 'DELETE'])
def set_folder(self, folder):
"""
Sets the folder where the files to serve are located.
"""
self.folder = folder
self.templates.directories[0] = folder
self.app.root_path = folder
def run(self, host='0.0.0.0', port=8080):
"""
Launch a development web server.
"""
waitress.serve(self.app, host=host, port=port)
def get(self, path):
"""
Get the content of a file, indentified by its path relative to the folder configured
in PyGreen. If the file extension is one of the extensions that should be processed
through Mako, it will be processed.
"""
data = self.app.test_client().get("/%s" % path).data
return data
def gen_static(self, output_folder):
"""
Generates a complete static version of the web site. It will stored in
output_folder.
"""
files = []
for l in self.file_listers:
files += l()
for f in files:
_logger.info("generating %s" % f)
content = self.get(f)
loc = os.path.join(output_folder, f)
d = os.path.dirname(loc)
if not os.path.exists(d):
os.makedirs(d)
with open(loc, "wb") as file_:
file_.write(content)
def __call__(self, environ, start_response):
return self.app(environ, start_response)
def cli(self, cmd_args=None):
"""
The command line interface of PyGreen.
"""
logging.basicConfig(level=logging.INFO, format='%(message)s')
parser = argparse.ArgumentParser(description='PyGreen, micro web framework/static web site generator')
subparsers = parser.add_subparsers(dest='action')
parser_serve = subparsers.add_parser('serve', help='serve the web site')
parser_serve.add_argument('-p', '--port', type=int, default=8080, help='port to serve on')
parser_serve.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
parser_serve.add_argument('-d', '--disable-templates', action='store_true', default=False, help='just serve static files, do not use Mako')
def serve():
if args.disable_templates:
self.template_exts = set([])
self.run(port=args.port)
parser_serve.set_defaults(func=serve)
parser_gen = subparsers.add_parser('gen', help='generate a static version of the site')
parser_gen.add_argument('output', help='folder to store the files')
parser_gen.add_argument('-f', '--folder', default=".", help='folder containing files to generate')
def gen():
self.gen_static(args.output)
parser_gen.set_defaults(func=gen)
args = parser.parse_args(cmd_args)
self.set_folder(args.folder)
print(parser.description)
print("")
args.func()
class PyGreen:
def __init__(self):
# the Bottle application
self.app = bottle.Bottle()
# a set of strings that identifies the extension of the files
# that should be processed using Mako
self.template_exts = set(["html"])
# the folder where the files to serve are located. Do not set
# directly, use set_folder instead
self.folder = "."
# the TemplateLookup of Mako
self.templates = TemplateLookup(directories=[self.folder],
imports=["from markdown import markdown"],
input_encoding='iso-8859-1',
collection_size=100,
)
# A list of regular expression. Files whose the name match
# one of those regular expressions will not be outputed when generating
# a static version of the web site
self.file_exclusion = [r".*\.mako", r"(^|.*\/)\..*"]
def is_public(path):
for ex in self.file_exclusion:
if re.match(ex,path):
return False
return True
def base_lister():
files = []
for dirpath, dirnames, filenames in os.walk(self.folder):
for f in filenames:
absp = os.path.join(dirpath, f)
path = os.path.relpath(absp, self.folder)
if is_public(path):
files.append(path)
return files
# A list of function. Each function must return a list of paths
# of files to export during the generation of the static web site.
# The default one simply returns all the files contained in the folder.
# It is necessary to define new listers when new routes are defined
# in the Bottle application, or the static site generation routine
# will not be able to detect the files to export.
self.file_listers = [base_lister]
@self.app.route('/', method=['GET', 'POST', 'PUT', 'DELETE'])
@self.app.route('/<path:path>', method=['GET', 'POST', 'PUT', 'DELETE'])
def hello(path="index.html"):
if path.split(".")[-1] in self.template_exts:
if self.templates.has_template(path):
t = self.templates.get_template(path)
data = t.render_unicode(pygreen=pygreen,request=bottle.request)
return data.encode(t.module._source_encoding)
elif is_public(path):
return bottle.static_file(path, root=self.folder)
return bottle.HTTPError(404,'File does not exist.')
def set_folder(self, folder):
"""
Sets the folder where the files to serve are located.
"""
self.folder = folder
self.templates.directories[0] = folder
def run(self, **kwargs):
"""
Launch a development web server.
"""
kwargs.setdefault("host", "0.0.0.0")
bottle.run(self.app, server="waitress", **kwargs)
def get(self, path):
"""
Get the content of a file, indentified by its path relative to the folder configured
in PyGreen. If the file extension is one of the extensions that should be processed
through Mako, it will be processed.
"""
handler = wsgiref.handlers.SimpleHandler(sys.stdin, sys.stdout, sys.stderr, {})
handler.setup_environ()
env = handler.environ
env.update({'PATH_INFO': "/%s" % path, 'REQUEST_METHOD': "GET"})
out = b"".join(pygreen.app(env, lambda *args: None))
return out
def gen_static(self, output_folder):
"""
Generates a complete static version of the web site. It will stored in
output_folder.
"""
files = []
for l in self.file_listers:
files += l()
for f in files:
_logger.info("generating %s" % f)
content = self.get(f)
loc = os.path.join(output_folder, f)
d = os.path.dirname(loc)
if not os.path.exists(d):
os.makedirs(d)
with open(loc, "wb") as file_:
file_.write(content)
def cli(self, cmd_args=None):
"""
The command line interface of PyGreen.
"""
logging.basicConfig(level=logging.INFO, format='%(message)s')
parser = argparse.ArgumentParser(description='PyGreen, micro web framework/static web site generator')
subparsers = parser.add_subparsers(dest='action')
parser_serve = subparsers.add_parser('serve', help='serve the web site')
parser_serve.add_argument('-p', '--port', type=int, default=8080, help='folder containg files to serve')
parser_serve.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
parser_serve.add_argument('-d', '--disable-templates', action='store_true', default=False, help='just serve static files, do not use invoke Mako')
def serve():
if args.disable_templates:
self.template_exts = set([])
self.run(port=args.port)
parser_serve.set_defaults(func=serve)
parser_gen = subparsers.add_parser('gen', help='generate a static version of the site')
parser_gen.add_argument('output', help='folder to store the files')
parser_gen.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
def gen():
self.gen_static(args.output)
parser_gen.set_defaults(func=gen)
args = parser.parse_args(cmd_args)
self.set_folder(args.folder)
args.func()
class Mailer:
def __init__(self):
self.template_lookup = TemplateLookup(
directories=[
os.path.join(os.path.dirname(__file__), 'mail_templates')
],
strict_undefined=True,
)
def connect(self) -> smtplib.SMTP:
if config.manager.mail.ssl:
port = 465
elif config.manager.mail.starttls:
port = 587
else:
port = 25
if config.manager.mail.port is not None:
port = config.manager.mail.port
if config.manager.mail.ssl:
keyfile = config.manager.mail.keyfile
certfile = config.manager.mail.certfile
context = ssl.create_default_context(
) if not keyfile and not certfile else None
mailer = smtplib.SMTP_SSL(config.manager.mail.host,
port,
keyfile=keyfile,
certfile=certfile,
context=context)
else:
mailer = smtplib.SMTP(config.manager.mail.host, port)
try:
if config.manager.mail.starttls:
keyfile = config.manager.mail.keyfile
certfile = config.manager.mail.certfile
context = ssl.create_default_context(
) if not keyfile and not certfile else None
mailer.starttls(keyfile=keyfile,
certfile=certfile,
context=context)
if config.manager.mail.user and config.manager.mail.password:
mailer.login(config.manager.mail.user,
config.manager.mail.password)
except BaseException:
mailer.close()
raise
return mailer
def async_mailer(self) -> aiosmtplib.SMTP:
if config.manager.mail.ssl:
port = 465
elif config.manager.mail.starttls:
port = 587
else:
port = 25
if config.manager.mail.port is not None:
port = config.manager.mail.port
return aiosmtplib.SMTP(
config.manager.mail.host,
port,
username=config.manager.mail.user,
password=config.manager.mail.password,
use_tls=config.manager.mail.ssl,
start_tls=config.manager.mail.starttls,
client_cert=config.manager.mail.certfile,
client_key=config.manager.mail.keyfile,
)
def _render_template(self, language: str, name: str,
**kwargs) -> Tuple[str, str]:
if language != 'en_us' and not self.template_lookup.has_template(
f'{language}/{name}'):
language = 'en_us'
template = self.template_lookup.get_template(f'{language}/{name}')
data = template.render(
config=config,
**kwargs,
)
return data.split('\n', 1)
def send_mail(self, language: str, name: str, to: str, context: dict):
html_title, html_data = self._render_template(language, name + '.html',
**context)
txt_title, txt_data = self._render_template(language, name + '.txt',
**context)
assert txt_title == html_title
message = MIMEMultipart('alternative')
message['Subject'] = txt_title
message.attach(MIMEText(html_data, 'html'))
message.attach(MIMEText(txt_data, 'plain'))
with self.connect() as connected_mailer:
connected_mailer.sendmail(config.manager.mail.sender, [to],
message.as_bytes())
async def async_send_mail(self, language: str, name: str, to: str,
context: dict):
html_title, html_data = self._render_template(language, name + '.html',
**context)
txt_title, txt_data = self._render_template(language, name + '.txt',
**context)
assert txt_title == html_title
message = MIMEMultipart('alternative')
message['Subject'] = txt_title
message.attach(MIMEText(html_data, 'html'))
message.attach(MIMEText(txt_data, 'plain'))
async with self.async_mailer() as connected_mailer:
await connected_mailer.sendmail(config.manager.mail.sender, [to],
message.as_bytes())
class PyBlue:
TEMPLATE_DIR = op.abspath(op.join(op.split(__file__)[0], "templates"))
def __init__(self):
# the Bottle application
self.app = bottle.Bottle()
# a set of strings that identifies the extension of the files
# that should be processed using Mako
self.template_exts = set([".html"])
# the folder where the files to serve are located. Do not set
# directly, use set_folder() instead
self.folder = "."
# the list of all crawled files, initialized in set_folder()
self.files = []
# should it recrawl directories on each request, used in serving
# can be turned off in the options
self.refresh = False
# the TemplateLookup of Mako
self.templates = TemplateLookup(directories=[self.folder, self.TEMPLATE_DIR],
imports=[ "from pyblue.utils import markdown", "from pyblue.utils import rst, asc"],
input_encoding='iso-8859-1',
collection_size=100,
)
# A list of regular expression. Files whose the name match
# one of those regular expressions will not be outputed when generating
# a static version of the web site
self.file_exclusion = [r".*\.mako", r".*\.py", r"(^|.*\/)\..*"]
def is_public(path):
for ex in self.file_exclusion:
if re.match(ex, path):
return False
return True
def base_lister():
files = []
for dirpath, dirnames, filenames in os.walk(self.folder):
filenames.sort()
for f in filenames:
absp = os.path.join(dirpath, f)
path = os.path.relpath(absp, self.folder)
files.append(path)
files = filter(is_public, files)
return files
# A list of function. Each function must return a list of paths
# of files to export during the generation of the static web site.
# The default one simply returns all the files contained in the folder.
# It is necessary to define new listers when new routes are defined
# in the Bottle application, or the static site generation routine
# will not be able to detect the files to export.
self.file_listers = [base_lister]
def file_renderer(path):
if is_public(path):
_, ext = os.path.splitext(path)
if ext in self.template_exts and self.templates.has_template(path):
f = File(fname=path, root=self.folder)
t = self.templates.get_template(path)
if self.refresh:
self.files = self.collect_files()
try:
data = t.render_unicode(p=self, f=f, u=utils)
# if it made it this far the user wants these to be rendered as html
if ext == '.md':
data = utils.markdown(data)
elif ext == '.rst':
data = utils.rst(data)
page = data.encode(t.module._source_encoding)
return page
except Exception as exc:
_logger.error("error %s generating page %s" % (exc, path))
return exceptions.html_error_template().render()
return bottle.static_file(path, root=self.folder)
return bottle.HTTPError(404, 'File does not exist.')
# The default function used to render files. Could be modified to change the way files are
# generated, like using another template language or transforming css...
self.file_renderer = file_renderer
self.app.route('/', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda: self.file_renderer('index.html'))
self.app.route('/<path:path>', method=['GET', 'POST', 'PUT', 'DELETE'])(lambda path: self.file_renderer(path))
def set_folder(self, folder):
"""
Sets the folder where the files to serve are located.
"""
self.folder = folder
self.templates.directories[0] = folder
if self.folder not in sys.path:
sys.path.append(self.folder)
# append more ignored patterns
ignore_file = op.join(self.folder, ".ignore")
_logger.info("checking optional ignore file %s" % ignore_file)
self.file_exclusion.extend(utils.parse_opt_file(ignore_file))
_logger.info("excluded files: %s" % self.file_exclusion)
# add more of the included extensions
include_file = op.join(self.folder, ".include")
_logger.info("checking optional include files %s" % include_file)
self.template_exts.update(utils.parse_opt_file(include_file))
_logger.info("template extensions: %s" % self.template_exts)
# collect the files
self.files = self.collect_files()
def run(self, host='0.0.0.0', port=8080):
"""
Launch a development web server.
"""
waitress.serve(self, host=host, port=port)
def get(self, f):
"""
Get the content of a file, indentified by its path relative to the folder configured
in PyGreen. If the file extension is one of the extensions that should be processed
through Mako, it will be processed.
"""
handler = wsgiref.handlers.SimpleHandler(sys.stdin, sys.stdout, sys.stderr, {})
handler.setup_environ()
env = handler.environ
env.update({'PATH_INFO': "/%s" % f.fname, 'REQUEST_METHOD': "GET"})
out = b"".join(self.app(env, lambda *args: None))
return out
@property
def settings(self):
m = __import__('settings', globals(), locals(), [])
return m
def link(self, start, name, text=''):
items = list(filter(lambda x: re.search(name, x.fname, re.IGNORECASE), self.files))
if not items:
f = self.files[0]
_logger.error("link name '%s' in %s does not match" % (name, start.fname))
return ("#", "Link pattern '%s' does not match!" % name)
else:
f = items[0]
if len(items) > 1:
_logger.warn("link name '%s' in %s matches more than one item %s" % (name, start.fname, items))
link, value = f.url(start, text=text)
return (link, value)
def toc(self, start, tag=None, match=None, is_image=False):
"Produces name, links pairs from file names"
if tag:
items = filter(lambda x: tag in x.meta['tags'], self.files)
else:
items = self.files
if match:
items = filter(lambda x: re.search(match, x.fname, re.IGNORECASE), self.files)
if is_image:
items = filter(lambda x: x.is_image, items)
if not items:
_logger.error("tag %s does not match" % tag)
urls = [f.url(start) for f in items]
return urls
def collect_files(self):
"""
Collects all files that will be parsed. Will also be available in main context.
TODO: this method crawls the entire directory tree each time it is accessed.
It is handy during development but very large trees may affect performance.
"""
files = []
for l in self.file_listers:
files += l()
files = list(map(lambda x: File(x, self.folder), files))
_logger.info("collected %s files" % len(files))
# apply sort order
decor = [(f.sortkey, f.name, f) for f in files]
decor.sort()
return [ f[2] for f in decor ]
def gen_static(self, output_folder):
"""
Generates a complete static version of the web site. It will stored in
output_folder.
"""
# this makes all files available in the template context
for f in self.files:
if f.skip_file:
_logger.info("skipping large file %s of %.1fkb" % (f.fname, f.size))
continue
_logger.info("generating %s" % f.fname)
content = self.get(f)
f.write(output_folder, content)
def __call__(self, environ, start_response):
return self.app(environ, start_response)
def cli(self, cmd_args=None):
"""
The command line interface of PyGreen.
"""
import pyblue
parser = argparse.ArgumentParser(description='PyBlue %s, static site generator' % pyblue.VERSION)
subparsers = parser.add_subparsers(dest='action')
parser_serve = subparsers.add_parser('serve', help='serve the web site')
parser_serve.add_argument('-p', '--port', type=int, default=8080, help='folder containg files to serve')
parser_serve.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
parser_serve.add_argument('-d', '--disable-templates', action='store_true', default=False,
help='just serve static files, do not use invoke Mako')
parser_serve.add_argument('-v', '--verbose', default=False, action="store_true", help='outputs more messages')
parser_serve.add_argument('-n', '--norefresh', default=False, action="store_true", help='do not refresh files on every request')
def serve():
if args.disable_templates:
self.template_exts = set([])
self.refresh = not args.norefresh
self.run(port=args.port)
parser_serve.set_defaults(func=serve)
parser_gen = subparsers.add_parser('gen', help='generate a static version of the site')
parser_gen.add_argument('output', help='folder to store the files')
parser_gen.add_argument('-f', '--folder', default=".", help='folder containg files to serve')
parser_gen.add_argument('-v', '--verbose', default=False, action="store_true", help='outputs more messages')
def gen():
self.gen_static(args.output)
def set_log_level(level):
logging.basicConfig(level=level, format='%(levelname)s\t%(module)s.%(funcName)s\t%(message)s')
parser_gen.set_defaults(func=gen)
print(parser.description)
args = parser.parse_args(cmd_args)
level = logging.DEBUG if args.verbose else logging.WARNING
set_log_level(level)
_logger.info("starting pyblue")
self.set_folder(args.folder)
args.func()
class PyGreen:
def __init__(self):
# the Bottle application
self.app = bottle.Bottle()
# a set of strings that identifies the extension of the files
# that should be processed using Mako
self.template_exts = set(["html"])
# the folder where the files to serve are located. Do not set
# directly, use set_folder instead
self.folder = "."
# the TemplateLookup of Mako
self.templates = TemplateLookup(
directories=[self.folder],
imports=["from markdown import markdown"],
input_encoding='iso-8859-1',
collection_size=100,
)
# A list of regular expression. Files whose the name match
# one of those regular expressions will not be outputed when generating
# a static version of the web site
self.file_exclusion = [r".*\.mako", r"(^|.*\/)\..*"]
def is_public(path):
for ex in self.file_exclusion:
if re.match(ex, path):
return False
return True
def base_lister():
files = []
for dirpath, dirnames, filenames in os.walk(self.folder):
for f in filenames:
absp = os.path.join(dirpath, f)
path = os.path.relpath(absp, self.folder)
if is_public(path):
files.append(path)
return files
# A list of function. Each function must return a list of paths
# of files to export during the generation of the static web site.
# The default one simply returns all the files contained in the folder.
# It is necessary to define new listers when new routes are defined
# in the Bottle application, or the static site generation routine
# will not be able to detect the files to export.
self.file_listers = [base_lister]
@self.app.route('/', method=['GET', 'POST', 'PUT', 'DELETE'])
@self.app.route('/<path:path>',
method=['GET', 'POST', 'PUT', 'DELETE'])
def hello(path="index.html"):
if path.split(".")[-1] in self.template_exts:
if self.templates.has_template(path):
t = self.templates.get_template(path)
data = t.render_unicode(pygreen=pygreen,
request=bottle.request)
return data.encode(t.module._source_encoding)
elif is_public(path):
return bottle.static_file(path, root=self.folder)
return bottle.HTTPError(404, 'File does not exist.')
def set_folder(self, folder):
"""
Sets the folder where the files to serve are located.
"""
self.folder = folder
self.templates.directories[0] = folder
def run(self, **kwargs):
"""
Launch a development web server.
"""
kwargs.setdefault("host", "0.0.0.0")
bottle.run(self.app, server="waitress", **kwargs)
def get(self, path):
"""
Get the content of a file, indentified by its path relative to the folder configured
in PyGreen. If the file extension is one of the extensions that should be processed
through Mako, it will be processed.
"""
handler = wsgiref.handlers.SimpleHandler(sys.stdin, sys.stdout,
sys.stderr, {})
handler.setup_environ()
env = handler.environ
env.update({'PATH_INFO': "/%s" % path, 'REQUEST_METHOD': "GET"})
out = b"".join(pygreen.app(env, lambda *args: None))
return out
def gen_static(self, output_folder):
"""
Generates a complete static version of the web site. It will stored in
output_folder.
"""
files = []
for l in self.file_listers:
files += l()
for f in files:
_logger.info("generating %s" % f)
content = self.get(f)
loc = os.path.join(output_folder, f)
d = os.path.dirname(loc)
if not os.path.exists(d):
os.makedirs(d)
with open(loc, "wb") as file_:
file_.write(content)
def cli(self, cmd_args=None):
"""
The command line interface of PyGreen.
"""
logging.basicConfig(level=logging.INFO, format='%(message)s')
parser = argparse.ArgumentParser(
description='PyGreen, micro web framework/static web site generator'
)
subparsers = parser.add_subparsers(dest='action')
parser_serve = subparsers.add_parser('serve',
help='serve the web site')
parser_serve.add_argument('-p',
'--port',
type=int,
default=8080,
help='folder containg files to serve')
parser_serve.add_argument('-f',
'--folder',
default=".",
help='folder containg files to serve')
parser_serve.add_argument(
'-d',
'--disable-templates',
action='store_true',
default=False,
help='just serve static files, do not use invoke Mako')
def serve():
if args.disable_templates:
self.template_exts = set([])
self.run(port=args.port)
parser_serve.set_defaults(func=serve)
parser_gen = subparsers.add_parser(
'gen', help='generate a static version of the site')
parser_gen.add_argument('output', help='folder to store the files')
parser_gen.add_argument('-f',
'--folder',
default=".",
help='folder containg files to serve')
def gen():
self.gen_static(args.output)
parser_gen.set_defaults(func=gen)
args = parser.parse_args(cmd_args)
self.set_folder(args.folder)
args.func()
class ConfigService(abc.ABC):
"""
Base class for creating configurable services.
"""
# validation period in seconds, how frequent validation is attempted
validation_period = 0.5
# time to wait in seconds for determining if service started successfully
validation_timer = 5
def __init__(self, node: CoreNode) -> None:
"""
Create ConfigService instance.
:param node: node this service is assigned to
"""
self.node = node
class_file = inspect.getfile(self.__class__)
templates_path = pathlib.Path(class_file).parent.joinpath(
TEMPLATES_DIR)
self.templates = TemplateLookup(directories=templates_path)
self.config = {}
self.custom_templates = {}
self.custom_config = {}
configs = self.default_configs[:]
self._define_config(configs)
@staticmethod
def clean_text(text: str) -> str:
"""
Returns space stripped text for string literals, while keeping space
indentations.
:param text: text to clean
:return: cleaned text
"""
return inspect.cleandoc(text)
@property
@abc.abstractmethod
def name(self) -> str:
raise NotImplementedError
@property
@abc.abstractmethod
def group(self) -> str:
raise NotImplementedError
@property
@abc.abstractmethod
def directories(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def files(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def default_configs(self) -> List[Configuration]:
raise NotImplementedError
@property
@abc.abstractmethod
def modes(self) -> Dict[str, Dict[str, str]]:
raise NotImplementedError
@property
@abc.abstractmethod
def executables(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def dependencies(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def startup(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def validate(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def shutdown(self) -> List[str]:
raise NotImplementedError
@property
@abc.abstractmethod
def validation_mode(self) -> ConfigServiceMode:
raise NotImplementedError
def start(self) -> None:
"""
Creates services files/directories, runs startup, and validates based on
validation mode.
:return: nothing
:raises ConfigServiceBootError: when there is an error starting service
"""
logging.info("node(%s) service(%s) starting...", self.node.name,
self.name)
self.create_dirs()
self.create_files()
wait = self.validation_mode == ConfigServiceMode.BLOCKING
self.run_startup(wait)
if not wait:
if self.validation_mode == ConfigServiceMode.TIMER:
self.wait_validation()
else:
self.run_validation()
def stop(self) -> None:
"""
Stop service using shutdown commands.
:return: nothing
"""
for cmd in self.shutdown:
try:
self.node.cmd(cmd)
except CoreCommandError:
logging.exception(
f"node({self.node.name}) service({self.name}) "
f"failed shutdown: {cmd}")
def restart(self) -> None:
"""
Restarts service by running stop and then start.
:return: nothing
"""
self.stop()
self.start()
def create_dirs(self) -> None:
"""
Creates directories for service.
:return: nothing
:raises CoreError: when there is a failure creating a directory
"""
for directory in self.directories:
try:
self.node.privatedir(directory)
except (CoreCommandError, ValueError):
raise CoreError(
f"node({self.node.name}) service({self.name}) "
f"failure to create service directory: {directory}")
def data(self) -> Dict[str, Any]:
"""
Returns key/value data, used when rendering file templates.
:return: key/value template data
"""
return {}
def set_template(self, name: str, template: str) -> None:
"""
Store custom template to render for a given file.
:param name: file to store custom template for
:param template: custom template to render
:return: nothing
"""
self.custom_templates[name] = template
def get_text_template(self, name: str) -> str:
"""
Retrieves text based template for files that do not have a file based template.
:param name: name of file to get template for
:return: template to render
"""
raise CoreError(f"service({self.name}) unknown template({name})")
def get_templates(self) -> Dict[str, str]:
"""
Retrieves mapping of file names to templates for all cases, which
includes custom templates, file templates, and text templates.
:return: mapping of files to templates
"""
templates = {}
for name in self.files:
basename = pathlib.Path(name).name
if name in self.custom_templates:
template = self.custom_templates[name]
template = self.clean_text(template)
elif self.templates.has_template(basename):
template = self.templates.get_template(basename).source
else:
template = self.get_text_template(name)
template = self.clean_text(template)
templates[name] = template
return templates
def create_files(self) -> None:
"""
Creates service files inside associated node.
:return: nothing
"""
data = self.data()
for name in self.files:
basename = pathlib.Path(name).name
if name in self.custom_templates:
text = self.custom_templates[name]
rendered = self.render_text(text, data)
elif self.templates.has_template(basename):
rendered = self.render_template(basename, data)
else:
text = self.get_text_template(name)
rendered = self.render_text(text, data)
logging.debug(
"node(%s) service(%s) template(%s): \n%s",
self.node.name,
self.name,
name,
rendered,
)
self.node.nodefile(name, rendered)
def run_startup(self, wait: bool) -> None:
"""
Run startup commands for service on node.
:param wait: wait successful command exit status when True, ignore status
otherwise
:return: nothing
:raises ConfigServiceBootError: when a command that waits fails
"""
for cmd in self.startup:
try:
self.node.cmd(cmd, wait=wait)
except CoreCommandError as e:
raise ConfigServiceBootError(
f"node({self.node.name}) service({self.name}) failed startup: {e}"
)
def wait_validation(self) -> None:
"""
Waits for a period of time to consider service started successfully.
:return: nothing
"""
time.sleep(self.validation_timer)
def run_validation(self) -> None:
"""
Runs validation commands for service on node.
:return: nothing
:raises ConfigServiceBootError: if there is a validation failure
"""
start = time.monotonic()
cmds = self.validate[:]
index = 0
while cmds:
cmd = cmds[index]
try:
self.node.cmd(cmd)
del cmds[index]
index += 1
except CoreCommandError:
logging.debug(f"node({self.node.name}) service({self.name}) "
f"validate command failed: {cmd}")
time.sleep(self.validation_period)
if cmds and time.monotonic() - start > self.validation_timer:
raise ConfigServiceBootError(
f"node({self.node.name}) service({self.name}) failed to validate"
)
def _render(self, template: Template, data: Dict[str, Any] = None) -> str:
"""
Renders template providing all associated data to template.
:param template: template to render
:param data: service specific defined data for template
:return: rendered template
"""
if data is None:
data = {}
return template.render_unicode(node=self.node,
config=self.render_config(),
**data)
def render_text(self, text: str, data: Dict[str, Any] = None) -> str:
"""
Renders text based template providing all associated data to template.
:param text: text to render
:param data: service specific defined data for template
:return: rendered template
"""
text = self.clean_text(text)
try:
template = Template(text)
return self._render(template, data)
except Exception:
raise CoreError(
f"node({self.node.name}) service({self.name}) "
f"{exceptions.text_error_template().render_unicode()}")
def render_template(self,
basename: str,
data: Dict[str, Any] = None) -> str:
"""
Renders file based template providing all associated data to template.
:param basename: base name for file to render
:param data: service specific defined data for template
:return: rendered template
"""
try:
template = self.templates.get_template(basename)
return self._render(template, data)
except Exception:
raise CoreError(
f"node({self.node.name}) service({self.name}) "
f"{exceptions.text_error_template().render_template()}")
def _define_config(self, configs: List[Configuration]) -> None:
"""
Initializes default configuration data.
:param configs: configs to initialize
:return: nothing
"""
for config in configs:
self.config[config.id] = config
def render_config(self) -> Dict[str, str]:
"""
Returns configuration data key/value pairs for rendering a template.
:return: nothing
"""
if self.custom_config:
return self.custom_config
else:
return {k: v.default for k, v in self.config.items()}
def set_config(self, data: Dict[str, str]) -> None:
"""
Set configuration data from key/value pairs.
:param data: configuration key/values to set
:return: nothing
:raise CoreError: when an unknown configuration value is given
"""
for key, value in data.items():
if key not in self.config:
raise CoreError(f"unknown config: {key}")
self.custom_config[key] = value
