def __init__(self, check, context):
self.check = check
self.env = ImmutableSandboxedEnvironment()
_ctx = copy.deepcopy(context)
self.variables = _ctx.pop("variables", {})
self.ctx = {"depc": _ctx}
def __init__(
self, string: str, context: Dict[str, Any] = {}, *args, **kwargs
) -> None:
assert isinstance(string, str), "Expression must be a string"
self.string = string
self.environment = ImmutableSandboxedEnvironment(undefined=StrictUndefined)
self.context = context
self.environment.globals = self.context
def __init__(self, args={}):
self.log = RevSysLogger(args["verbose_debug"])
self.nginx_conf = {}
self._set_default_values()
self._interpret_arguments(args)
self.env = ImmutableSandboxedEnvironment(line_statement_prefix=None,
trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined)
class Expression(object):
def __init__(
self, string: str, context: Dict[str, Any] = {}, *args, **kwargs
) -> None:
assert isinstance(string, str), "Expression must be a string"
self.string = string
self.environment = ImmutableSandboxedEnvironment(undefined=StrictUndefined)
self.context = context
self.environment.globals = self.context
def compile(self) -> bool:
result = False
try:
expr = self.environment.compile_expression(
self.string, undefined_to_none=False
)
result = expr()
if result:
# needs to evaluate it to trigger undefined error
pass
except UndefinedError as e:
raise ExpressionException(e.message)
except TemplateSyntaxError as e:
raise ExpressionException(e.message)
except Exception as e:
raise ExpressionException(str(e))
# if str(e) == "argument of type 'StrictUndefined' is not iterable":
# raise ExpressionException(str(e))
# else:
# raise e
return result
def create_sandboxed_environment(*, loader: Optional[BaseLoader]=None) \
-> Environment:
"""Create a sandboxed environment."""
if loader is None:
# A loader that never finds a template.
loader = FunctionLoader(lambda name: None)
return ImmutableSandboxedEnvironment(loader=loader, autoescape=True)
def render(template: str, context: Dict[str, Any] = {}) -> str:
environment = ImmutableSandboxedEnvironment(undefined=StrictUndefined)
environment.globals = context
jinja_template = environment.from_string(template)
output = None
try:
output = jinja_template.render()
except UndefinedError as e:
raise TemplateException(e.message)
except TypeError as e:
if str(e) == "no loader for this environment specified":
raise TemplateException("Extending templates is not allowed")
else:
raise e
return output
def create_sandboxed_env():
"""Create a sandboxed environment."""
# A loader that never finds a template.
dummy_loader = FunctionLoader(lambda name: None)
return ImmutableSandboxedEnvironment(loader=dummy_loader,
autoescape=True,
lstrip_blocks=True,
trim_blocks=True)
def jinja_render(content, context):
if not content:
content = {}
from jinja2.runtime import Undefined
env = ImmutableSandboxedEnvironment(
loader=DjangoLoader(),
cache_size=0,
undefined=Undefined,
)
context.update(default_jinja_context)
try:
return env.get_template(content).render(context)
except Exception as e:
logger.debug('----- render content failed -----')
logger.debug(content)
logger.debug('--------------- end -------------')
import traceback
traceback.print_exc()
raise
class Template(object):
def __init__(self, check, context):
self.check = check
self.env = ImmutableSandboxedEnvironment()
_ctx = copy.deepcopy(context)
self.variables = _ctx.pop("variables", {})
self.ctx = {"depc": _ctx}
def _recursive_render(self, tpl):
"""
A variable can contain itself another variable : we loop until
the template is finally completely formed.
"""
new_tpl = self.env.from_string(tpl).render(**self.ctx)
if new_tpl != tpl:
return self._recursive_render(new_tpl)
else:
return tpl
def _render_part(self, value):
"""
Recursively parse a dict and render every part.
"""
if isinstance(value, str):
return self._recursive_render(value)
elif isinstance(value, list):
for i, l in enumerate(value):
value[i] = self._render_part(l)
elif isinstance(value, dict):
for k, v in value.items():
value[k] = self._render_part(v)
return value
def render(self, context=None):
if context:
self.ctx["depc"] = context
# Add custom filters
self.env.filters["iso8601"] = iso8601_filter
self.env.filters["regex_search"] = regex_search_filter
self.env.filters["regex_search_bool"] = regex_search_bool_filter
self.env.filters["bool"] = to_bool_filter
# Add the variables
self.ctx["depc"]["rule"] = self.variables.get("rule", {})
self.ctx["depc"]["team"] = self.variables.get("team", {})
self.ctx["depc"]["check"] = self.variables.get("checks", {}).get(
self.check.name, {})
self.ctx["depc"]["source"] = self.variables.get("sources", {}).get(
self.check.source.name, {})
parameters = copy.deepcopy(self.check.parameters)
return self._render_part(parameters)
class Templates:
"""Helper for rendering templates."""
_templates_dict: Dict[str, str]
_environment: ImmutableSandboxedEnvironment
def __init__(self, templates_dict: Dict[str, str]):
self._templates_dict = templates_dict
self._environment = ImmutableSandboxedEnvironment(loader=DictLoader(
self._templates_dict),
enable_async=True)
def __setitem__(self, key: str, template: str):
self._templates_dict[key] = template
def __delitem__(self, key: str):
del self._templates_dict[key]
def _html_template_out_of_parameters(self, parameters: Dict[str, Any]):
body = "\n".join(
"<dt>{key}</dt><dd>{{{{ {key} }}}}</dd>".format(key=key)
for key in parameters.keys())
return f'<!DOCTYPE html><html><head><meta charset="UTF-8"></head><body><dl>{body}</dl></body></html>'
async def _render_without_template(self, parameters: Dict[str,
Any]) -> str:
template_id = str(uuid4())
self[template_id] = self._html_template_out_of_parameters(parameters)
rendered = await self.render(template_id, parameters)
del self[template_id]
return rendered
async def render(self, template_id: str, parameters: Dict[str,
Any]) -> str:
"""Render the template with given template_id
Args:
template_id (str): template id
parameters (Dict[str, Any]): dictionary of variables to be put in the template
Returns:
str: [description]
"""
try:
template = self._environment.get_template(template_id)
return await template.render_async(**parameters)
except TemplateNotFound:
new_params = {"missing_template_id": template_id, **parameters}
return await self._render_without_template(new_params)
def create_sandbox_env():
"""Build an safe environment for rendering emails
We want non-developers to be able to write emails, but using the main
templating system would grant them access to secrets as well as the
ability to run arbitrary code. ImmutableSandboxedEnvironment gets us
most of the way there, but doesn't know which config is sensitive,
or whether globals can be safely accessed.
To avoid being annoyingly different, we try to match what Flask
does in `create_jinja_environment`, but with a pared-down set of
config and globals.
"""
default_jinja_options = {}
if app.jinja_options != default_jinja_options:
# This code doesn't support any unusual options yet. If we've
# (say) added an extension to the main template system, it should
# be allowed or ignored here.
raise NotImplementedError
# Don't autoescape because this is used to generate plaintext output
env = ImmutableSandboxedEnvironment(autoescape=False)
config_to_copy = [
"DEBUG",
"SERVER_NAME",
]
config = {c: app.config[c] for c in config_to_copy if c in app.config}
# We don't need things like request and session for emails
env.globals.update(
url_for=url_for,
config=config,
)
return env
def test_immutable_environment(self):
env = ImmutableSandboxedEnvironment()
self.assert_raises(SecurityError,
env.from_string('{{ [].append(23) }}').render)
self.assert_raises(SecurityError,
env.from_string('{{ {1:2}.clear() }}').render)
def test_immutable_environment(self):
env = ImmutableSandboxedEnvironment()
self.assert_raises(SecurityError, env.from_string('{{ [].append(23) }}').render)
self.assert_raises(SecurityError, env.from_string('{{ {1:2}.clear() }}').render)
def test_immutable_environment(self, env):
env = ImmutableSandboxedEnvironment()
pytest.raises(SecurityError,
env.from_string("{{ [].append(23) }}").render)
pytest.raises(SecurityError,
env.from_string("{{ {1:2}.clear() }}").render)
def __init__(self, templates_dict: Dict[str, str]):
self._templates_dict = templates_dict
self._environment = ImmutableSandboxedEnvironment(loader=DictLoader(
self._templates_dict),
enable_async=True)
from collections import namedtuple
from distutils.version import LooseVersion
from operator import attrgetter
from jinja2.sandbox import ImmutableSandboxedEnvironment
from cumulusci.core.config import TaskConfig
from cumulusci.core.config import FlowConfig
from cumulusci.core.exceptions import FlowConfigError, FlowInfiniteLoopError
from cumulusci.core.utils import import_global
# TODO: define exception types: flowfailure, taskimporterror, etc?
RETURN_VALUE_OPTION_PREFIX = "^^"
jinja2_env = ImmutableSandboxedEnvironment()
class StepVersion(LooseVersion):
"""Like LooseVersion, but converts "/" into -1 to support comparisons"""
def parse(self, vstring: str):
super().parse(vstring)
self.version = tuple(-1 if x == "/" else x for x in self.version)
class StepSpec(object):
""" simple namespace to describe what the flowrunner should do each step """
__slots__ = (
"step_num", # type: str
"task_name", # type: str
class NginxConfigSDK:
"""NginxConfigSDK refreshes the configuration of the Nginx server based on
default or given values.
Args:
args (dict, optional): A dictionary of configuration parameters to
overide default action of class. Default is empty.
Attributes:
log (instance): Logging utility.
nginx_conf (dict): Configuration parameter dictionary.
env (instance): Jinja Sandbox enivornment.
"""
def __init__(self, args={}):
self.log = RevSysLogger(args["verbose_debug"])
self.nginx_conf = {}
self._set_default_values()
self._interpret_arguments(args)
self.env = ImmutableSandboxedEnvironment(line_statement_prefix=None,
trim_blocks=True,
lstrip_blocks=True,
undefined=StrictUndefined)
# Sets default location of JSON and Jinja template
def _set_default_values(self):
self.nginx_conf["jinja_template"] = script_configs.JINJA_TEMPLATE
self.nginx_conf["jinja_conf_vars"] = script_configs.JINJA_CONF_VARS
self.nginx_conf["conf_name"] = script_configs.CONF_NAME
self.nginx_conf["tmp_location"] = script_configs.TMP_PATH
self.nginx_conf["final_location"] = script_configs.NGINX_FINAL_LOCATION
self.nginx_conf[
"backup_location"] = script_configs.NGINX_BACKUP_LOCATION
# Reads the options provided by the command line call
def _interpret_arguments(self, args):
# override local arguments if a value was provided from outside
for item in args:
self.nginx_conf[item] = args[item]
# Read contents of Jinja template
def _read_jinja_template(self):
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
template_full_path = self.nginx_conf["jinja_template"]
self.log.LOGD("Loading SDK Nginx template!")
with open(template_full_path) as f:
self.string_template = f.read()
self.log.LOGD("Loaded SDK Nginx template: " + template_full_path)
# Read JSON file and save them into class instance's config_vars variable
def _read_sdk_config_files(self):
"""
input: reads the content of the configuration json file in
class variable nginx_conf["jinja_conf_vars"]
output: returns a status code and populates self.config_vars if everything is ok
- 0 - if no problems have been encountered
- 1 - if the provided file is not in the correct json file
- 2 - if the provided file doesn't exist
- 3 - unknown error case
"""
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
conf_full_path = self.nginx_conf["jinja_conf_vars"]
self.log.LOGD("Reading file from: " + conf_full_path)
try:
with open(conf_full_path) as f:
try:
self.config_vars = json.load(f)
return 0
except BaseException:
self.log.LOGE("Bad JSON format for file " + conf_full_path)
return 1
except BaseException:
self.log.LOGE("Can't find file " + conf_full_path)
return 2
# Generates Nginx config file from jinja template stored in class variable
# string_template and writes into final file.
def _generate_final_nginx_config(self):
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
key_list = self.config_vars["configs"]
template = self.env.from_string(self.string_template)
final_nginx_config = template.render(
configs=key_list, bpname=socket.gethostname().split('.')[0])
final_file = self.nginx_conf["tmp_location"] + \
self.nginx_conf["conf_name"]
with open(final_file, 'w+') as f:
f.write(final_nginx_config)
shutil.copy2(
self.nginx_conf["tmp_location"] + self.nginx_conf["conf_name"],
self.nginx_conf["final_location"] + self.nginx_conf["conf_name"])
# Backs up the current Nginx apps Configuration found in
# /etc/nginx/conf.d/revsw-apps.conf on edge server.
def _backup_active_sdk_nginx_config(self):
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
# make backup for this file
conf_final_path = self.nginx_conf["final_location"] + \
self.nginx_conf["conf_name"]
conf_backup_path = self.nginx_conf["backup_location"] + \
self.nginx_conf["conf_name"]
try:
if not os.path.exists(self.nginx_conf["backup_location"]):
os.makedirs(self.nginx_conf["backup_location"])
if os.path.exists(conf_final_path):
shutil.copy2(conf_final_path, conf_backup_path)
except BaseException:
self.log.LOGE(
"An error appeared while trying to backup the original file " +
conf_final_path + " to " + conf_backup_path)
raise
# Remove the current Nginx Configuration
def _remove_active_sdk_nginx_config(self):
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
# remove the active configuration file
conf_final_path = self.nginx_conf["final_location"] + \
self.nginx_conf["conf_name"]
try:
if os.path.exists(conf_final_path):
# remove the file
os.remove(conf_final_path)
except BaseException:
self.log.LOGE(
"An error appeared while removing the configuration file " +
conf_final_path)
raise
# Restores the backup Nginx configuration
def _restore_sdk_nginx_from_backup(self):
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
# restore file from tmp backup location
try:
shutil.copy2(
self.nginx_conf["backup_location"] +
self.nginx_conf["conf_name"],
self.nginx_conf["final_location"] +
self.nginx_conf["conf_name"])
except BaseException:
self.log.LOGE(
"An error appeared while trying to get backup file! Stop processing"
)
raise
# Reloads the running Nginx process
def _load_new_configuration(self):
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
# check if configuration is correct
p = subprocess.Popen('/etc/init.d/revsw-nginx configtest', shell=True)
p.communicate()
if p.returncode != 0:
self.log.LOGE("Nginx configuration has a problem!")
return p.returncode
# nginx reload configuration if there are no errors
p = subprocess.Popen('/etc/init.d/revsw-nginx reload', shell=True)
p.communicate()
return p.returncode
def refresh_configuration(self):
"""Refreshes Nginx configuration based on Jinja template and JSON found in
jinja_template and jinja_conf_vars class variables
"""
self.log.LOGI("Starting processing " + sys._getframe().f_code.co_name)
# backup current configuration
self._read_jinja_template()
file_read_status = self._read_sdk_config_files()
# in case that a problem appeared reading the configuration JSON file
# remove the config file and reload nginx
if file_read_status != 0:
self._remove_active_sdk_nginx_config()
result = self._load_new_configuration()
if result != 0:
exit(1)
exit(0)
flags_problem = 0
try:
# check if configuration type is correct
tmp = self.config_vars["configuration_type"]
if tmp != "sdk_apps_config":
self.log.LOGE("The provided configuration type is not valid!")
raise
except BaseException:
self.log.LOGE("Key configuration_type must be defined!")
flags_problem = 1
try:
# check if operation is defined
tmp = self.config_vars["operation"]
except BaseException:
self.log.LOGE("JSON file doesn't contain operation type!")
flags_problem = 1
try:
# check configs is defined
tmp = self.config_vars["configs"]
except BaseException:
self.log.LOGE("JSON file doesn't contain configs parameter!")
flags_problem = 1
if flags_problem == 1:
self._remove_active_sdk_nginx_config()
result = self._load_new_configuration()
exit(1)
if self.config_vars["operation"] != "app-update":
self.log.LOGD(
"Unknown operation was provided! Exiting gracefully!")
exit(0)
config_problem = 0
try:
if not isinstance(self.config_vars["configs"], list):
self.log.LOGE("Param configs should be a list!")
raise
if len(self.config_vars["configs"]) < 1:
self.log.LOGE("At least one config should be defined!")
raise
except BaseException:
config_problem = 1
if config_problem == 0:
self._backup_active_sdk_nginx_config()
self._generate_final_nginx_config()
else:
self._remove_active_sdk_nginx_config()
result = self._load_new_configuration()
if (result != 0) and (config_problem == 0):
self.log.LOGE(
"Problem loading new configuration - restoring original file")
self._restore_sdk_nginx_from_backup()
sys.exit(1)