def init_gunicorn():
global mc
middlewares = []
middlewares.append(db_handler())
if settings.debug:
middlewares.append(session_middleware(SimpleCookieStorage()))
else:
middlewares.append(
session_middleware(EncryptedCookieStorage(settings.session_key)))
app = web.Application(middlewares=middlewares)
aiohttp_jinja2.setup(app, loader=jinja2.FunctionLoader(load_templ))
# Memcache init
app.mc = aiomcache.Client(settings.memcache['addr'],
settings.memcache['port'])
# Mongo init
db_connect(app)
union_routes(os.path.join(settings.tao_path, 'libs'))
union_routes(os.path.join(settings.root_path, 'apps'))
union_routes(os.path.join(settings.root_path), p=True)
for res in routes:
name = res[3]
if name is None: name = '{}:{}'.format(res[0], res[2])
app.router.add_route(res[0], res[1], res[2], name=name)
path = os.path.join(settings.root_path, 'static')
app.router.add_static('/static/', path, name='static')
return app
def __init__(self, *args, sql=None, executor=None, **kwargs):
if not executor:
executor = DebugSQLExecutor()
elif isinstance(executor, str):
executor = SQLExecutor.for_kind(executor)()
elif issubclass(executor, SQLExecutor):
executor = executor()
MeasureProvider.__init__(self, *args, **kwargs)
self._base_sql = textwrap.dedent(sql).strip() if sql else None
self.executor = executor
self.dialect = DIALECTS[executor.dialect]
self.provides_measure('count',
shared=True,
distribution=None,
default=0)
self._template_environment = jinja2.Environment(
loader=jinja2.FunctionLoader(lambda x: x),
undefined=jinja2.StrictUndefined)
self._template_environment.filters.update({
'col': self._col,
'val': self._val
})
def _get_value_render_env(self):
if self._env is None:
env = jinja2.Environment(loader=jinja2.FunctionLoader(load_func))
self._env = env
else:
env = self._env
return env
def __init__(self, app):
self.app = app
self.templates = {}
global jinja_env
jinja_env = jinja2.Environment(
loader=jinja2.FunctionLoader(self.get_template_body))
def _RenderTemplate(template_subdir, template_name, **context):
"""Loads a template file and renders it to unicode.
Args:
template_subdir: The subdirectory in gae/templates containing the template
file.
template_name: The name of the template file.
**context: Optional key/value pairs to render into the template.
Returns:
The given template file rendered with the given context as a unicode string.
Raises:
jinja2.TemplateNotFound: if the given template file doesn't exist.
"""
# Create a partial loading function, which will return the contents of the
# template given just the template name.
loading_func = functools.partial(_LoadTemplate, template_subdir)
# Construct an Environment and retrieve the Template.
env = jinja2.Environment(loader=jinja2.FunctionLoader(loading_func),
autoescape=True,
extensions=['jinja2.ext.autoescape'],
finalize=lambda value: value or '',
variable_start_string='[[',
variable_end_string=']]',
undefined=jinja2.StrictUndefined)
template = env.get_template(template_name)
# Render the template with the provided context.
return template.render(**context)
def update_amrfinderplus_singularity(amrfinderplus_version, db_version, author, config):
'''
Use the config.toml, or override any of the options via the command line
'''
# load the params
config = toml.load(config)
if amrfinderplus_version is not None:
config['amrfinderplus_version'] = mlst_version
if author is not None:
config['author'] = author
if db_version:
config['db_version'] = False
# load the template
loader = jinja2.FunctionLoader(load_template)
env = jinja2.Environment(loader=loader)
SINGULARITY_RECIPE = env.get_template("amrfinderplus.singularity").render(config)
# prepare the folders
version_path = pathlib.Path(f'v{config["amrfinderplus_version"]}')
if not version_path.exists():
version_path.mkdir()
today = pendulum.today().format('YYYYMMDD')
subfolder_path = version_path / today
if not subfolder_path.exists():
subfolder_path.mkdir()
# create local version
local_recipe = subfolder_path / \
f'Singularity.v{config["amrfinderplus_version"]}_{today}'
local_recipe.write_text(SINGULARITY_RECIPE)
# create global version
global_recipe = pathlib.Path("Singularity")
global_recipe.write_text(SINGULARITY_RECIPE)
def read_config_file(path, variables={}):
""" reads yaml config file, passes it through jinja and returns data structre
- OS ENV are available as {{ ENV.<VAR> }}
- variables defined in parent configs are available as {{ <VAR> }}
"""
jinja_variables = copy.deepcopy(variables)
jinja_variables['ENV'] = os.environ
if path.exists():
logger.debug("Reading config file: {}".format(path))
# First check for sops being present
try:
jenv = jinja2.Environment(
enable_async=True,
auto_reload=False,
loader=jinja2.FunctionLoader(_sops_loader),
undefined=jinja2.StrictUndefined,
extensions=['jinja2.ext.loopcontrols'])
template = jenv.get_template(str(path))
rendered_template = template.render(jinja_variables)
data = yaml.safe_load(rendered_template)
if data:
return data
except Exception as e:
logger.exception("Error reading config file: {} ({})".format(
path, e))
sys.exit(1)
return {}
def run(template_path, data):
env = jinja2.Environment(
loader=jinja2.FunctionLoader(_load_template),
undefined=jinja2.StrictUndefined,
)
t = env.get_or_select_template(template_path)
return t.render(**data)
def __init__(self, storage, macro_default_page):
self._storage = storage
self._macro_default_page = macro_default_page
self._macro_import = '{{% import "{}" as m %}}\n'.format(
macro_default_page)
self._env = jinja2.Environment(loader=jinja2.FunctionLoader(
self._load_jinja_template()),
extensions=["jinja2.ext.loopcontrols"],
autoescape=False,
cache_size=0)
self._md = markdown.Markdown(extensions=[
"markdown.extensions.attr_list", "markdown.extensions.fenced_code",
"markdown.extensions.footnotes", "markdown.extensions.tables",
"markdown.extensions.toc", "markdown.extensions.wikilinks"
],
extension_configs={
"markdown.extensions.wikilinks": {
"build_url": self._build_url,
"base_url": "/p/",
"end_url": ""
}
},
output_format="html5")
def render_template(self, template, **args):
"""Find and render a template."""
env = jinja2.Environment(
loader=jinja2.FunctionLoader(self.get_template))
tpl = env.get_template(template)
return tpl.render(**args)
def GetJinjaEnv():
"""Gets a jinja2 Environment which includes the desired path.
Returns:
A jinja2 Environment.
"""
loader = jinja2.FunctionLoader(_LoadTemplateAsResource)
return jinja2.Environment(
loader=loader, autoescape=True, extensions=['jinja2.ext.autoescape'],
finalize=lambda value: value or '', variable_start_string='[[',
variable_end_string=']]')
def create_celery_configuration(ctx, runner, agent_config, resource_loader):
create_celery_includes_file(ctx, runner, agent_config)
loader = jinja2.FunctionLoader(resource_loader)
env = jinja2.Environment(loader=loader)
config_template_path = get_agent_resource_local_path(
ctx, agent_config, 'celery_config_path')
config_template = env.get_template(config_template_path)
config_template_values = {
'includes_file_path':
agent_config['includes_file'],
'celery_base_dir':
agent_config['celery_base_dir'],
'worker_modifier':
agent_config['name'],
'management_ip':
utils.get_manager_ip(),
'broker_ip':
utils.get_manager_ip(),
'agent_ip':
get_agent_ip(ctx, agent_config),
'celery_user':
agent_config['user'],
'celery_group':
agent_config['user'],
'worker_autoscale':
'{0},{1}'.format(agent_config['max_workers'],
agent_config['min_workers'])
}
ctx.logger.debug(
'Populating celery config jinja2 template with the following '
'values: {0}'.format(config_template_values))
config = config_template.render(config_template_values)
init_template_path = get_agent_resource_local_path(ctx, agent_config,
'celery_init_path')
init_template = env.get_template(init_template_path)
init_template_values = {
'celery_base_dir': agent_config['celery_base_dir'],
'worker_modifier': agent_config['name']
}
ctx.logger.debug(
'Populating celery init.d jinja2 template with the following '
'values: {0}'.format(init_template_values))
init = init_template.render(init_template_values)
ctx.logger.debug(
'Creating celery config and init files [cloudify_agent={0}]'.format(
agent_config))
runner.put(agent_config['config_file'], config, use_sudo=True)
runner.put(agent_config['init_file'], init, use_sudo=True)
def _make_environment(load, additionals, extensions):
extensions.append(ext.with_)
env = ResolvingByRelativePathEnvironment(
loader=jinja2.FunctionLoader(load),
undefined=jinja2.StrictUndefined,
trim_blocks=False,
lstrip_blocks=True,
extensions=extensions,
)
for name, defs in additionals.items():
getattr(env, name).update(defs)
return env
def render_template(template, *args, **kw):
if os.path.exists(template):
def load_template(template):
with open(template, 'r') as f:
return f.read()
loader = jinja2.FunctionLoader(load_template)
else:
loader = jinja2.FileSystemLoader(config.template_dir)
env = jinja2.Environment(loader=loader)
formatters.apply(env, *formatters.all)
template = env.get_template(template)
return template.render(*args, **kw)
def load_project(self, project):
self.project = self._load_project_from_dir(project)
if not self.project:
self.project = AbstractProject()
self.project.directory = project
def file_load(name):
"""
The default FileSystemLoader doesn't load files specified
by absolute paths or paths that include '..' - therefore
we provide a custom fallback function that handles this.
"""
name = os.path.abspath(name)
try:
with open(name, 'rb') as f:
return f.read().decode('utf-8')
except Exception:
raise jinja2.TemplateNotFound(name)
def string_load(name):
"""
Allow specifying a string instead of template to be able
to return expanded config/specs/...
"""
if name.startswith(('{{', '{%')):
return name
raise jinja2.TemplateNotFound(name)
loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader(self.pm_tpl.get_path()),
jinja2.FunctionLoader(string_load),
jinja2.FunctionLoader(file_load),
])
self.project.tplgen = jinja2.Environment(loader=loader,
**self.jinjaenv_args)
self.project.abstract_initialize()
def _create_jinja_environment(runfiles, site_root, link_ext):
def _Load(path):
return runfiles.Rlocation(
posixpath.join(WORKSPACE_DIR, TEMPLATE_PATH, path))
env = jinja2.Environment(loader=jinja2.FunctionLoader(_Load),
keep_trailing_newline=True,
line_statement_prefix='%')
env.filters['markdown'] = lambda text: jinja2.Markup(mistune.markdown(text)
)
env.filters['doc_link'] = (
lambda fname: site_root + '/' + fname + '.' + link_ext)
env.filters['link'] = lambda fname: site_root + '/' + fname
return env
def render_scenario(self):
"""Render a scenario and save to disk."""
j2_env = j2.Environment(loader=j2.FunctionLoader(
self.get_template), trim_blocks=True, undefined=j2.StrictUndefined)
j2_env.filters['env_override'] = filters.env_override
template = j2_env.get_template(self.scenario_fpath)
try:
rendered_scenario = template.render(vars=self.vars)
except j2.exceptions.UndefinedError as e:
LOG.error(e)
missing_arg = re.findall(
r'no attribute (.*)', e.message)[0].strip("'")
LOG.error(usage_exc.missing_arg(missing_arg))
sys.exit(2)
self.write_rendered_scenario(rendered_scenario)
def bootstrap():
"""Load the TEMPLATE_ENVIRONMENT global variable."""
from django.conf import settings
if not settings.configured:
# At least this will make it work, even if it's using the defaults.
settings.configure()
from djanjinja import bccache
from djanjinja.extensions.cache import CacheExtension
# Get the bytecode cache object.
bytecode_cache = bccache.get_cache()
default_extensions = set(
['jinja2.ext.do', 'jinja2.ext.loopcontrols', CacheExtension])
if getattr(settings, 'USE_I18N', False):
default_extensions.add('jinja2.ext.i18n')
autoescape = getattr(settings, 'JINJA_AUTOESCAPE', False)
if autoescape:
default_extensions.add('jinja2.ext.autoescape')
extensions = list(
set(getattr(settings, 'JINJA_EXTENSIONS',
[])).union(default_extensions))
# Set up global `TEMPLATE_ENVIRONMENT` variable.
global TEMPLATE_ENVIRONMENT
TEMPLATE_ENVIRONMENT = Environment(
loader=jinja2.FunctionLoader(get_template_source),
auto_reload=getattr(settings, 'DEBUG', True),
autoescape=autoescape,
bytecode_cache=bytecode_cache,
extensions=extensions)
if getattr(settings, 'USE_I18N', False):
# The `django.utils.translation` module behaves like a singleton of
# `gettext.GNUTranslations`, since it exports all the necessary
# methods.
from django.utils import translation
# pylint: disable-msg=E1101
TEMPLATE_ENVIRONMENT.install_gettext_translations(translation)
bundles = getattr(settings, 'DJANJINJA_BUNDLES', [])
for bundle_specifier in bundles:
app_label, bundle_name = bundle_specifier.rsplit('.', 1)
TEMPLATE_ENVIRONMENT.load(app_label, bundle_name)
def setup_pp_env(page=None, page_filters=[]):
remote, path = get_page_where(page)
if remote:
logger.debug("Using remote template loader for page %s" % page)
pp_env = jinja2.Environment(
loader=jinja2.FunctionLoader(read_markdown_remote))
else:
logger.debug("Using FileSystemLoader for page %s" % page)
pp_env = jinja2.Environment(loader=jinja2.FileSystemLoader(path))
# Pull exported values (& functions) from page filters into the pp_env
for filter_name in page_filters:
if "export" in dir(filters[filter_name]):
for key, val in filters[filter_name].export.items():
logger.debug("... pulling in filter_%s's exported key '%s'" %
(filter_name, key))
pp_env.globals[key] = val
return pp_env
def setup_pp_env(page=None,
page_filters=[],
no_loader=False,
strict_undefined=False):
remote, path = get_page_where(page)
if strict_undefined:
preferred_undefined = jinja2.StrictUndefined
else:
preferred_undefined = jinja2.Undefined
if remote:
logger.debug("Using remote template loader for page %s" % page)
pp_env = jinja2.Environment(
undefined=preferred_undefined,
loader=jinja2.FunctionLoader(read_markdown_remote))
elif no_loader:
logger.debug("Using a no-loader Jinja environment")
pp_env = jinja2.Environment(undefined=preferred_undefined)
else:
logger.debug("Using FileSystemLoader for page %s" % page)
pp_env = jinja2.Environment(undefined=preferred_undefined,
loader=jinja2.FileSystemLoader(path))
# Add custom "defined_and_" tests
def defined_and_equalto(a, b):
return pp_env.tests["defined"](a) and pp_env.tests["equalto"](a, b)
pp_env.tests["defined_and_equalto"] = defined_and_equalto
def undefined_or_ne(a, b):
return pp_env.tests["undefined"](a) or pp_env.tests["ne"](a, b)
pp_env.tests["undefined_or_ne"] = undefined_or_ne
# Pull exported values (& functions) from page filters into the pp_env
for filter_name in page_filters:
if filter_name not in config.filters.keys():
logger.debug("Skipping unloaded filter '%s'" % filter_name)
continue
if "export" in dir(config.filters[filter_name]):
for key, val in config.filters[filter_name].export.items():
logger.debug("... pulling in filter_%s's exported key '%s'" %
(filter_name, key))
pp_env.globals[key] = val
return pp_env
def _ensure_env(self, env: Union[jinja2.Environment, None]):
"""
Make sure the jinja environment is minimally configured.
"""
if not env:
env = jinja2.Environment()
if not env.loader:
env.loader = jinja2.FunctionLoader(
lambda filename: self._cache[filename])
if 'faker' not in env.globals:
faker = Faker()
faker.seed(1234)
env.globals['faker'] = faker
if 'random_model' not in env.globals:
env.globals['random_model'] = jinja2.contextfunction(random_model)
if 'random_models' not in env.globals:
env.globals['random_models'] = jinja2.contextfunction(
random_models)
return env
def update_meningotype_singularity(version, mlst_version, author, config):
'''
Use the config.toml, or override any of the options via the command line
'''
# load the params
config = toml.load(config)
if version is not None:
config['version'] = version
if mlst_version is not None:
config['mlst_version'] = mlst_version
if author is not None:
config['author'] = author
# load the template
loader = jinja2.FunctionLoader(load_template)
env = jinja2.Environment(loader=loader)
SINGULARITY_RECIPE = env.get_template("_singularity.j2").render(config)
# create global version
global_recipe = pathlib.Path("Singularity")
global_recipe.write_text(SINGULARITY_RECIPE)
def update_singularity(version, author, config):
'''
update the singularity recipe for new version of roary
'''
config = toml.load('config.toml')
if version is not None:
config['version'] = version
if author is not None:
config['author'] = author
loader = jinja2.FunctionLoader(load_template)
env = jinja2.Environment(loader=loader)
SINGULARITY_RECIPE = env.get_template("roary.singularity").render(config)
# create global version
global_recipe = pathlib.Path("Singularity")
global_recipe.write_text(SINGULARITY_RECIPE)
def __init__(self,
*args,
sql=None,
db_client=None,
dialect='presto',
**kwargs):
assert db_client is not None, "Must specify an (Omniduct-compatible) database client."
MeasureProvider.__init__(self, *args, **kwargs)
self._base_sql = textwrap.dedent(sql).strip() if sql else None
self.db_client = db_client
self.dialect = DIALECTS[dialect]
self.provides_measure('count', shared=True, distribution=None)
self._template_environment = jinja2.Environment(
loader=jinja2.FunctionLoader(lambda x: x))
self._template_environment.filters.update({
'col': self._col,
'val': self._val
})
def _ensure_env(self, env: Union[jinja2.Environment, None]):
"""
Make sure the jinja environment is minimally configured.
"""
if not env:
env = jinja2.Environment()
if not env.loader:
env.loader = jinja2.FunctionLoader(
lambda path: self._file_cache[path])
if 'faker' not in env.globals:
faker = Faker()
faker.seed_instance(1234)
env.globals['faker'] = faker
env.globals.setdefault('hash_password', hash_password)
env.globals.setdefault('random_model',
jinja2.contextfunction(random_model))
env.globals.setdefault('random_models',
jinja2.contextfunction(random_models))
return env
def _render_template(self, tmpl, data):
jtmpl = self._env.get_template(basename(tmpl.src))
try:
s = jtmpl.render(**data)
except SkipGeneration:
return
dst = tmpl.dst
if dst:
if '{' in dst:
env = jinja2.Environment(
loader=jinja2.FunctionLoader(lambda _: dst),
undefined=jinja2.StrictUndefined,
)
dst = env.get_template('_').render(**data)
with open(dst, 'w', encoding='utf-8') as fp:
fp.write(s)
else:
print(s)
def render_jinja(dict_, template_str):
"""Render dict onto jinja template and return the string result"""
name = 'jvars'
def loader(n):
if n == name:
return template_str
else:
kkdir = pathfinder.PathFinder.find_kolla_kubernetes_dir()
path = os.path.join(kkdir, n)
f = open(path)
d = f.read()
f.close
return d
j2env = jinja2.Environment(loader=jinja2.FunctionLoader(loader))
# Do not print type for bools "!!bool" on output
j2env.filters['bool'] = TypeUtils.str_to_bool
j2env.filters['min'] = min
j2env.filters['b64encode'] = lambda x: \
base64.b64encode(x.encode('ascii'))
j2env.filters['b64decode'] = base64.b64decode
j2env.filters['to_nice_yaml'] = lambda x: \
yaml.safe_dump(x, default_flow_style=False) if x else ''
# Add a "raise" keyword for raising exceptions from within jinja
def jinja_raise(message):
raise Exception(message)
j2env.globals['raise'] = jinja_raise
# Add a keyword for accessing KubeUtils from within jinja
j2env.globals['KubeUtils'] = KubeUtils
# Render the template
rendered_template = j2env.get_template(name).render(dict_)
return rendered_template + "\n"
def load_jupyter_server_extension(nbapp):
# Add our custom templates to the beginning of the search path.
templates_dir = pathlib.Path(__file__).parent.absolute().joinpath("templates").as_posix()
# NbApp settings have already been propogated so we need to modify the webapp settings directly
# nbapp.extra_template_paths.append(templates_dir)
env = nbapp.web_app.settings["jinja2_env"]
if nbapp.__class__.__name__ == 'SingleUserNotebookApp':
# Running in jupyterhub.
# jupyterhub-singleuser already extends 'templates/page.html'. Since we
# want to extend it as well, we have to do something funky here.
# We are patching this logic:
# https://github.com/jupyterhub/jupyterhub/blob/9391470e269cf2cbc39520f903ab42900aa991ca/jupyterhub/singleuser.py#L483-L492
# So that
# our page.html --extends-> singleuser.page_template --extends-> templates/page.html
from jupyterhub import singleuser
import jinja2
def get_page(name):
if name == 'singleuser.page_template':
return singleuser.page_template
env.loader = jinja2.ChoiceLoader([
jinja2.FileSystemLoader(os.path.join(templates_dir, 'jupyterhub-singleuser')),
jinja2.FileSystemLoader(templates_dir),
jinja2.FunctionLoader(get_page),
env.loader
])
pass
else:
env.loader.searchpath.insert(0, templates_dir)
nbapp.log.info("Imandra theme enabled!")
def generate_code(self):
self.prepare_model()
jinja_env = jinja2.Environment(
loader=jinja2.FunctionLoader(load_template),
trim_blocks=True,
lstrip_blocks=True)
jinja_env.filters['sqliteType'] = _sqliteType
jinja_env.filters['sqliteConstraint'] = _sqliteConstraint
jinja_env.filters['strftime'] = lambda x: x.strftime(
"%d.%m.%Y %H:%M:%S")
jinja_env.filters['primaryKeyProp'] = _getPrimaryKeyProperty
template = jinja_env.get_template('sqlite.jinja')
if not exists(join(self.srcgen_folder, 'sql')):
mkdir(join(self.srcgen_folder, 'sql'))
with open(join(self.srcgen_folder, 'sql/schema.sql'), 'w') as f:
f.write(
template.render(application=self.model,
timestamp=datetime.now(),
model_filename=self.model_filename))
}
def _load_template(filename):
if filename in MATTA_TEMPLATE_FILES:
filename = '{0}/templates/{1}'.format(SRC_DIR, filename)
with open(filename, 'r') as f:
code = f.read()
# the lambda avoids template caching
return (code, filename, lambda *a, **k: False)
SRC_DIR = os.path.dirname(os.path.realpath(__file__))
env = jinja2.environment.Environment()
env.loader = jinja2.FunctionLoader(_load_template)
class MattaJSONEncoder(json.JSONEncoder):
"""
A Pandas/numpy/networkx aware JSON Encoder.
Based on http://stackoverflow.com/questions/3488934/simplejson-and-numpy-array
"""
def default(self, obj):
if isinstance(obj, np.ndarray):
return obj.tolist()
elif isinstance(obj, np.generic):
return obj.item()
elif isinstance(obj, nx.Graph) or isinstance(obj, nx.DiGraph):
if nx.is_tree(obj) and 'root' in obj.graph:
