def run(self):
if len(self._undefined_vars) > 0:
if self._configuration["task_run_mode"] == "background":
raise JinjamatorTaskRunException(
"cannot run task because of undefined variables: {0}".
format(self._undefined_vars))
else:
for var in self._undefined_vars:
self.handle_undefined_var(var)
schema = self.get_jsonform_schema()["schema"]
# del schema["properties"]["custom_parameters"]
# del schema["properties"]["output_plugin"]
# del schema["properties"]["task"]
# validate_jsonschema(instance=self.configuration._data, schema=schema)
# validate_jsonschema(instance=self.configuration._data, schema=self._output_plugin.get_json_schema(self.configuration._data)['schema'])
results = []
to_process = copy.copy(self._tasklets)
for tasklet in self._tasklets:
self._global_ldr = init_loader(self)
for content_plugin_dir in self._configuration.get(
"global_content_plugins_base_dirs", []):
self._global_ldr.load(f"{content_plugin_dir}")
if os.path.isdir(self._configuration["taskdir"] +
"/plugins/content"):
self._log.debug("found task plugins directory " +
self._configuration["taskdir"] +
"/plugins/content")
self._global_ldr.load(self._configuration["taskdir"] +
"/plugins/content")
self._current_tasklet = tasklet
retval = ""
self._log.debug("running with dataset: \n{0}".format(
json.dumps(self.configuration._data, indent=2)))
if tasklet.endswith("j2"):
try:
template_string = ""
with open(tasklet) as fh:
tmp = fh.read().split("\n")
try:
if tmp[0][:2] == "#!":
del tmp[0]
except IndexError:
pass
template_string = "\n".join(tmp)
t = self.j2_environment.from_string(template_string)
retval = t.render(self.configuration)
except jinja2.exceptions.UndefinedError as e:
self._log.error("{0}\n{1} -> skipping tasklet {2}".format(
traceback.format_exc(), e.message, tasklet))
except IndexError as e:
self._log.error("{0}\n{1} -> skipping tasklet {2}".format(
traceback.format_exc(), e.message, tasklet))
pass
elif tasklet.endswith("py"):
task_code = self.get_py_tasklet_code(tasklet)
# self._log.debug(task_code)
module = import_code(task_code, tasklet[:-3].replace("/", "."))
for k, v in iteritems(self.configuration._data):
setattr(module, k, v)
setattr(module, "__file__", tasklet)
setattr(module.jinjaTask, "parent", self)
setattr(module.jinjaTask, "configuration",
self.configuration._data)
setattr(module.jinjaTask, "_configuration",
self._configuration._data)
module._log = self._log
try:
retval = module.jinjaTask().__run__()
except Exception as e:
_, _, tb = sys.exc_info()
error_text = ""
code_to_show = ""
for filename, lineno, funname, line in reversed(
traceback.extract_tb(tb)):
if filename == "<string>":
filename = tasklet
if filename == tasklet and funname == "__run__":
try:
code_to_show = (
"[ line " + str(lineno - 7) + " ] " +
task_code.split("\n")[lineno - 1])
error_text += f"{e}\n{filename}:{lineno -7}, in {funname}\n {line}\n {code_to_show}\n\n"
except Exception as inner_e:
self._log.debug(
f"cannot show code -> this is a bug \n{inner_e }\n task_code: {task_code}\nlineno: {lineno}"
)
else:
error_text += (
f"{e}\n{filename}:{lineno}, in {funname}\n {line}\n"
)
self._log.error(error_text)
if self.configuration.get("best_effort"):
continue
else:
to_process.pop(0)
skipped = []
for path in to_process:
skipped.append(os.path.basename(path))
results.append({
"tasklet_path": tasklet,
"result": "",
"status": "error",
"error": error_text,
"skipped": skipped,
})
raise TaskletFailed(
results,
f"tasklet {tasklet} has failed and best_effort is not defined -> exiting",
)
else:
raise ValueError(
f"tasklet {tasklet} has file extension which is not supported"
)
self._output_plugin.init_plugin_params()
self._output_plugin.connect()
self._output_plugin.process(retval,
template_path=tasklet,
current_data=self.configuration)
results.append({
"tasklet_path": tasklet,
"result": retval,
"status": "ok",
"error": "",
"skipped": [],
})
to_process.pop(0)
if self._configuration["task_run_mode"] == "background":
self._log.tasklet_result(
"{0}".format(retval)) # this submits the result via celery
return results
def setup(app):
from jinjamator.plugin_loader.content import (
ContentPluginLoader,
global_ldr,
init_loader,
contentPlugin,
)
from pprint import pprint
globals()["global_ldr"] = init_loader(app)
globals()["global_ldr"].load(
"/home/putzw/git/jinjamator/jinjamator/plugins/content"
)
import importlib
import traceback
import warnings
from typing import Any, Callable, Dict, List, Mapping, NamedTuple, Optional, Tuple
from sphinx.deprecation import RemovedInSphinx40Warning, deprecated_alias
from sphinx.pycode import ModuleAnalyzer
from sphinx.util import logging
from sphinx.util.inspect import isclass, isenumclass, safe_getattr
logger = logging.getLogger(__name__)
import sphinx.ext.autodoc
from sphinx.ext.autodoc.importer import import_module, mangle, unmangle
def import_object_new(
modname: str,
objpath: List[str],
objtype: str = "",
attrgetter: Callable[[Any, str], Any] = safe_getattr,
warningiserror: bool = False,
) -> Any:
import imp
import types
print(
"----------------------------------------hack---------------------------------"
)
# print(f'modname: {modname} objpath: {objpath}')
# module = imp.new_module(modname)
global_ldr = globals()["global_ldr"]
class_path = modname.split(".")
base_obj_name = class_path.pop(0)
base_obj = global_ldr.get_functions()[base_obj_name]
# print(global_ldr.get_functions())
module = imp.new_module(base_obj_name)
# sys.modules[base_obj_name] = module
# for path in class_path:
# cur = getattr(base_obj,item)
parent = base_obj
# base_obj=getattr(base_obj,path)
for item in dir(base_obj):
cur = getattr(base_obj, item)
if isinstance(cur, contentPlugin):
setattr(module, item, cur)
print(f"register {base_obj_name}.{item}")
return [module, None, modname, module]
# parent=None
# for path in class_path:
# parent=base_obj
# base_obj=getattr(base_obj,path)
# pprint(dir(base_obj))
# func = getattr(module, func_name)
# if isinstance(func, types.FunctionType):
if objpath:
logger.debug("[autodoc] from %s import %s", modname, ".".join(objpath))
else:
logger.debug("[autodoc] import %s", modname)
try:
# module = None
exc_on_importing = None
objpath = list(objpath)
while module is None:
try:
module = import_module(modname, warningiserror=warningiserror)
logger.debug("[autodoc] import %s => %r", modname, module)
except ImportError as exc:
logger.debug("[autodoc] import %s => failed", modname)
exc_on_importing = exc
if "." in modname:
# retry with parent module
modname, name = modname.rsplit(".", 1)
objpath.insert(0, name)
else:
raise
obj = module
parent = None
object_name = None
for attrname in objpath:
parent = obj
logger.debug("[autodoc] getattr(_, %r)", attrname)
mangled_name = mangle(obj, attrname)
obj = attrgetter(obj, mangled_name)
logger.debug("[autodoc] => %r", obj)
object_name = attrname
print([module, parent, object_name, obj])
return [module, parent, object_name, obj]
except (AttributeError, ImportError) as exc:
if isinstance(exc, AttributeError) and exc_on_importing:
# restore ImportError
exc = exc_on_importing
if objpath:
errmsg = "autodoc: failed to import %s %r from module %r" % (
objtype,
".".join(objpath),
modname,
)
else:
errmsg = "autodoc: failed to import %s %r" % (objtype, modname)
if isinstance(exc, ImportError):
# import_module() raises ImportError having real exception obj and
# traceback
real_exc, traceback_msg = exc.args
if isinstance(real_exc, SystemExit):
errmsg += (
"; the module executes module level statement "
"and it might call sys.exit()."
)
elif isinstance(real_exc, ImportError) and real_exc.args:
errmsg += (
"; the following exception was raised:\n%s" % real_exc.args[0]
)
else:
errmsg += (
"; the following exception was raised:\n%s" % traceback_msg
)
else:
errmsg += (
"; the following exception was raised:\n%s" % traceback.format_exc()
)
logger.debug(errmsg)
raise ImportError(errmsg) from exc
def load(self, path):
self.task_base_dir = path
self._unresolved_task_base_dir = path
self._log.debug(f"---------------- load task: {path} ----------------")
search_paths = self._configuration.get("global_tasks_base_dirs", [])
self._log.debug(f"task search paths are: {search_paths}")
if not os.path.isabs(path):
for global_base_dir in search_paths:
tried_path = os.path.join(global_base_dir, path)
if os.path.exists(tried_path):
self._log.debug(f"resolved path to {tried_path}")
self._current_global_base_dir = global_base_dir
path = tried_path
break
if os.path.isfile(path):
self.task_base_dir = os.path.dirname(path)
self._tasklets = [path]
path = self.task_base_dir
self._configuration["taskdir"] = self.task_base_dir
elif os.path.isdir(path):
self.task_base_dir = path
self._configuration["taskdir"] = path
for file_type in self._supported_file_types:
self._tasklets.extend(
glob.glob("{0}/{1}".format(self.task_base_dir, file_type)))
else:
raise ValueError(f"cannot load path {path}")
self._global_ldr = init_loader(self)
if os.path.isdir(self._configuration["taskdir"] + "/plugins/content"):
self._log.debug("found task plugins directory " +
self._configuration["taskdir"] +
"/plugins/content")
self._global_ldr.load(self._configuration["taskdir"] +
"/plugins/content")
for content_plugin_dir in self._configuration.get(
"global_content_plugins_base_dirs", []):
self._global_ldr.load(f"{content_plugin_dir}")
self.setup_jinja2()
self._tasklets = natsorted(self._tasklets)
try:
self._default_values = self.configuration.merge_yaml(
"{0}/defaults.yaml".format(path),
private_data=self._configuration._data)
self._log.debug("loaded {0}/defaults.yaml".format(path))
for var in self._undefined_vars:
if var in self.configuration._data:
self._undefined_vars.remove(var)
if var in self._configuration._data:
self._undefined_vars.remove(var)
except FileNotFoundError:
pass
except jinja2.exceptions.TemplateNotFound:
pass
if self.configuration["undo"]:
self.is_undo_run = True
self._tasklets.reverse()
self._log.info("undo run detected: reversing task order")
else:
self.is_undo_run = False
if not self._tasklets:
raise ValueError("no task items found")
self.analyze_tasklets()
from flask import request
from jwt import InvalidSignatureError, ExpiredSignatureError
from functools import wraps
import random
from pprint import pformat
from glob import glob
import os
from copy import deepcopy
import logging
import string
log = logging.getLogger()
aaa_providers = {}
_global_ldr = init_loader(None)
from sqlalchemy import or_, and_
class AuthProviderBase(object):
def __init__(self, app=None):
self._app = app
self._log = logging.getLogger()
self._user = None
self._redirect_uri = None
self.static_users = []
self.static_roles = []
def register(self, **kwargs):
self._log.info("not implemented")