def main(self):
for d in [
"environments",
"logs",
"tasks",
"uploads",
"aaa",
"conf.d",
"resources/python",
]:
os.makedirs(
os.path.sep.join(
[self._configuration["jinjamator_user_directory"], d]),
exist_ok=True,
)
sys.path.insert(
0,
os.path.join(self._configuration["jinjamator_user_directory"],
"resources/python"),
)
if self._configuration["daemonize"]:
from jinjamator.daemon import run as app_run
app_run(self._configuration)
else:
# legacy cli task
from jinjamator.task import JinjamatorTask
task = JinjamatorTask("interactive")
if self._configuration["global_defaults"]:
task.configuration.merge_yaml(
self._configuration["global_defaults"])
task.configuration.merge_dict(self.configuration)
task._configuration.merge_dict(self._configuration)
task.load_output_plugin(
self.configuration["output_plugin"],
self._configuration["global_output_plugins_base_dirs"],
)
try:
task.load(self._configuration["taskdir"])
except ValueError:
if os.path.isdir(self._configuration["taskdir"]):
self._log.error(
f'No Tasklets found in {self._configuration["taskdir"]} -> exiting'
)
else:
self._log.error(
f'Task directory {self._configuration["taskdir"]} not found -> exiting'
)
task.run()
def run(path, task_data=False, **kwargs):
"""calls another jinjamator task"""
if path == "../":
tmp = _jinjamator.task_base_dir.split(os.path.sep)
if tmp[0] == "":
tmp[0] = os.path.sep
path = os.path.join(*tmp[:-1])
parent_data = copy.deepcopy(_jinjamator.configuration._data)
parent_private_data = copy.deepcopy(_jinjamator._configuration._data)
output_plugin = (
kwargs.get("output_plugin", False)
or parent_data.get("output_plugin", False)
or "console"
)
task = JinjamatorTask(parent_private_data.get("task_run_mode"))
task._configuration.merge_dict(parent_private_data)
task._parent_tasklet = _jinjamator._current_tasklet
if parent_private_data.get("task_run_mode") == "background":
backup = task._log.handlers[1].formatter._task
task._parent_tasklet = backup._current_tasklet
task._parent_task_id = id(backup)
task._log.handlers[1].formatter._task = task
if task_data:
task.configuration.merge_dict(
task_data,
dict_strategy="merge",
list_strategy="override",
other_types_strategy="override",
type_conflict_strategy="override",
)
else:
task.configuration.merge_dict(parent_data)
task.configuration["output_plugin"] = output_plugin
task._configuration["global_tasks_base_dirs"].insert(0, _jinjamator.task_base_dir)
task.load(path)
task.load_output_plugin(
output_plugin, task._configuration.get("global_output_plugins_base_dirs")
)
retval = task.run()
if parent_private_data.get("task_run_mode") == "background":
task._log.handlers[1].formatter._task = backup
task._parent_tasklet = backup._parent_tasklet
del task
return retval
def run_jinjamator_task(self, path, data, output_plugin):
"""
Jinjamator Celery Task runner.
"""
self.update_state(
state="PROGRESS",
meta={
"status": "setting up jinjamator task run",
"configuration": {
"root_task_path": path
},
},
)
formatter = CeleryLogFormatter()
log_handler = CeleryLogHandler()
log_handler.setLevel(logging.DEBUG)
log_handler.setFormatter(formatter)
log_handler.set_celery_task(self)
log_handler.formatter.set_root_task_path(path)
if "jinjamator_pre_run_tasks" in data:
for pre_run_task in data["jinjamator_pre_run_tasks"]:
task = JinjamatorTask()
task._configuration._data["jinjamator_job_id"] = self.request.id
log_handler.formatter.set_jinjamator_task(task)
task._scheduler = self
task._log.addHandler(log_handler)
task._log.setLevel(logging.DEBUG)
if "output_plugin" in pre_run_task["task"]:
task.load_output_plugin(pre_run_task["task"]["output_plugin"])
else:
task.load_output_plugin("console")
task.configuration.merge_dict(pre_run_task["task"]["data"])
task._configuration.merge_dict(
celery.conf["jinjamator_private_configuration"])
task.configuration.merge_dict(deepcopy(data))
task.load(pre_run_task["task"]["path"])
task._log.info("running pre run task {}".format(
pre_run_task["task"]["path"]))
if not task.run():
raise Exception("task failed")
task._log.handlers.remove(log_handler)
log_handler._task = None
del task
self.update_state(
state="PROGRESS",
meta={
"status": "running main task",
"configuration": {
"root_task_path": path
}
},
)
task = JinjamatorTask()
task._configuration._data["jinjamator_job_id"] = self.request.id
task._scheduler = self
log_handler.formatter.set_jinjamator_task(task)
task._log.setLevel(logging.DEBUG)
task._log.addHandler(log_handler)
task.load_output_plugin(
output_plugin,
celery.conf["jinjamator_private_configuration"]
["global_output_plugins_base_dirs"],
)
task._configuration.merge_dict(
celery.conf["jinjamator_private_configuration"])
task.configuration.merge_dict(data)
task.load(path)
if not task.run():
raise Exception("task failed")
return {
"status": "finished task",
"stdout": task._stdout.getvalue(),
"log": log_handler.contents,
}