class ProcessGenericListView(ProcessSecurity, ListView):
title = ''
permissions = []
paginate_by = get_conf('views__paginate')
template_name = get_conf('views__templates__objects_list')
filters = {}
def get_filters_from_request(self):
# convert set to python dict
request_dict = {k: v for k, v in self.request.GET.lists()}
# search filters in url if filters do exists change key add __in
return {f'{k}__in': request_dict.get(k) for k in self.filters if request_dict.get(k)}
def get_queryset(self):
filters = self.get_filters_from_request()
if filters:
return self.model.objects.filter(**filters)
else:
return self.model.objects.all()
def get_context_data(self, *args, **kwargs):
context = super().get_context_data(**kwargs)
context['title'] = self.title
return context
class JobListView(ProcessGenericListView):
model = Job
title = 'Jobs'
filters = get_conf('views__job__list__url_allow_filters')
permissions = get_conf('views__job__list__permissions')
def post(self, request, *args, **kwargs):
request = JobCancelView.as_view()(request)
return self.get(request, *args, **kwargs)
class ProcessListView(ProcessGenericListView):
model = Process
title = 'Processes'
filters = get_conf('views__process__list__url_allow_filters')
permissions = get_conf('views__process__list__permissions')
def post(self, request, *args, **kwargs):
request = ProcessRunOnDemandView.as_view()(request)
return self.get(request, *args, **kwargs)
class JobTaskListView(ProcessGenericListView):
model = JobTask
title = 'JobTasks'
filters = get_conf('views__jobtask__list__url_allow_filters')
permissions = get_conf('views__jobtask__list__permissions')
@method_decorator(csrf_exempt)
def dispatch(self, request, *args, **kwargs):
return super().dispatch(request, args, kwargs)
def post(self, request, *args, **kwargs):
request = JobTaskManagementView.as_view()(request)
return self.get(request, *args, **kwargs)
def get_task_as_node(t):
job_task = True if isinstance(t, JobTask) else False
node_task = t.task if job_task else t
color = get_conf(
f'diagram__tasks_color__{t.status}') if job_task else get_conf(
'diagram__tasks_color__default')
return {
'id': node_task.name,
'name': node_task.name,
'title': t.title if job_task else node_task.description,
'level': node_task.level,
'offset': node_task.offset,
'info': t.info if job_task else node_task.description,
'color': color,
}
def diagram(obj):
try:
assert isinstance(obj, Process), "err"
process = obj
except AssertionError:
assert isinstance(
obj, Job), "argument is not Process object either Job object"
process = obj.process
data = []
nodes = []
# noinspection PyUnresolvedReferences
for task in obj.tasks.all():
nodes.append(get_task_as_node(task))
tk = task if isinstance(obj, Process) else task.task
if tk.childs.all().count():
for child in tk.childs.all():
data.append([str(tk.name), str(child.task.name)])
else:
data.append([str(tk.name), str(tk.name)])
data.sort()
response = html.replace('{id}', str(process.id))
response = response.replace('{name}', str(obj.__str__()))
response = response.replace('{data}', str(data))
response = response.replace('{nodes}', str(nodes))
response = response.replace(
'{chart_height}',
str(process.chart_height) or get_conf('diagram__chart_height'))
return mark_safe(response)
def run(self):
try:
try:
# get interpreter the default is the one used to run django
if not self.obj.task.interpreter:
cmd = [sys.executable]
else:
cmd = self.obj.task.interpreter.split()
# noinspection SpellCheckingInspection
if self.obj.task.code.file.__class__.__name__ == 'S3Boto3StorageFile':
if not os.path.isdir('/tmp/dj_process_tasks'):
os.makedirs('/tmp/dj_process_tasks')
file_path = os.path.join('/tmp',
self.obj.task.code.file.name)
if not os.path.isfile(file_path):
with open(file_path, 'wb') as code:
code.write(self.obj.task.code.file.read())
else:
file_path = self.obj.task.code.path
# append task file path and arguments if they exists
cmd.append(file_path)
if self.obj.task.arguments:
cmd += self.obj.task.arguments.split()
logger.info(f'command to execute {cmd}')
p = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
# return code must be 0 for success
self.obj.observations = stdout.decode('utf-8')
if p.returncode:
raise Exception(stderr.decode('utf-8'))
self.obj.set_status(JobTask.finished)
except Exception as e:
# if error then send to logger and also mark task and it's job as error
self.obj.observations += f"\nexception when running task {e}"
self.obj.set_status(JobTask.error)
self.obj.job.status = Job.error
logger.error(
f'task {self.obj} finished with error {self.obj.observations}'
)
self.obj.job.save()
# if there is a custom handler send task id and exception to it
error_handler_func = get_conf('task__error_handler')
logger.info(
f'sending info to handler {error_handler_func.__name__}')
error_handler_func(self.obj, e)
self.obj.dt_end = timezone.now()
self.obj.save()
except Exception as e:
logger.exception(f'error {e} when processing task {self.obj}')
class ProcessSecurity(LoginRequiredMixin, UserPassesTestMixin):
raise_exception = get_conf('views__security_raise_exception')
def test_func(self):
if hasattr(self, 'permissions'):
usr = self.request.user
[logger.debug(f'user {usr.username} permission {i} result {usr.has_perm(i)}') for i in self.permissions]
return not any([not self.request.user.has_perm(i) for i in self.permissions])
return True
class TaskCreateView(ProcessGenericCreateView):
model = Task
success_url = get_conf('views__task__create__success_url')
success_message = get_conf('views__task__create__success_message')
permissions = get_conf('views__task__create__permissions')
redirect_to_edit = get_conf('views__task__create__redirect_to_edit')
def post(self, request, *args, **kwargs):
self.object = None
form_class = self.get_form_class()
form = form_class(request.POST, request.FILES)
if form.is_valid():
obj = form.save()
messages.success(request, self.success_message)
if not self.redirect_to_edit:
return HttpResponseRedirect(self.get_success_url())
else:
return redirect('process-tasks-update', pk=obj.id)
return self.render_to_response(self.get_context_data(form=form))
class ProcessRunOnDemandView(ProcessSecurity, View):
permissions = get_conf('views__process__run__permissions')
def post(self, request, *args, **kwargs):
try:
process = get_object_or_404(Process,
id=self.request.POST['process'])
job, tasks = Job.create(process)
except Exception as e:
messages.error(request, _(f'{e}'))
finally:
return request
class DiagramView(ProcessSecurity, View):
model = None
template = get_conf('views__templates__object_diagram')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.permissions = ['view_jobs'
] if self.model == Job else ['view_processes']
def get(self, request, pk, *args, **kwargs):
obj = self.model.objects.get(id=pk)
return render(request, self.template, {'object': obj})
class ProcessGenericEditView(ProcessSecurity, SuccessMessageMixin):
fields = '__all__'
template_name = get_conf('views__templates__object_edit')
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.success_url = reverse_lazy(self.success_url)
def get_context_data(self, *args, **kwargs):
context = super().get_context_data(*args, **kwargs)
context['operation'] = self.operation
return context
def post(self, request, *args, **kwargs):
try:
logger.debug(f'cancel job i got post request=> {self.request.POST}')
task = get_object_or_404(JobTask, id=self.request.POST['task'])
action = self.request.POST['action']
if action == JobTask.reopened:
task.reopen(main=True)
else:
task.set_status(action)
messages.success(request, get_conf('views__jobtask__management__success_message').format(action=action))
except Exception as e:
messages.error(request, _(f'{e}'))
finally:
return request
def post(self, request, *args, **kwargs):
logger.debug(f'cancel job i got post request=> {self.request.POST}')
try:
job = get_object_or_404(Job, id=self.request.POST['job'])
except KeyError:
messages.error(request, _('job does not exists'))
return request
try:
job.cancel()
messages.success(request,
get_conf('views__job__cancel__success_message'))
except Exception as e:
logger.exception(f"can't cancel job due to error=> {e}")
messages.error(request, _(f'{e}'))
return request
class JobTaskManagementView(ProcessSecurity, View):
permissions = get_conf('views__jobtask__management__permissions')
# noinspection PyUnusedLocal
def post(self, request, *args, **kwargs):
try:
logger.debug(f'cancel job i got post request=> {self.request.POST}')
task = get_object_or_404(JobTask, id=self.request.POST['task'])
action = self.request.POST['action']
if action == JobTask.reopened:
task.reopen(main=True)
else:
task.status = action
task.save()
messages.success(request, get_conf('views__jobtask__management__success_message').format(action=action))
except Exception as e:
messages.error(request, _(f'{e}'))
finally:
return request
class JobCancelView(ProcessSecurity, View):
permissions = get_conf('views__job__cancel__permissions')
# noinspection PyUnusedLocal
def post(self, request, *args, **kwargs):
logger.debug(f'cancel job i got post request=> {self.request.POST}')
try:
job = get_object_or_404(Job, id=self.request.POST['job'])
except KeyError:
messages.error(request, _('job does not exists'))
return request
try:
job.cancel()
messages.success(request,
get_conf('views__job__cancel__success_message'))
except Exception as e:
logger.exception(f"can't cancel job due to error=> {e}")
messages.error(request, _(f'{e}'))
return request
class JobTaskDeleteView(ProcessGenericDeleteView):
model = JobTask
success_url = get_conf('views__jobtask__delete__success_url')
success_message = get_conf('views__jobtask__delete__success_message')
permissions = get_conf('views__jobtask__delete__permissions')
Highcharts.chart('container-process-{id}', {
chart: {
height: {chart_height},
inverted: true
},
title: {
useHTML: true,
text: '{name}'
},
series: [{
type: 'organization',
name: '{name}',
keys: ['from', 'to'],
data: {data},
levels: [],
linkRadius: """ + get_conf('diagram__link_radius') + """,
linkLineWidth: """ + get_conf('diagram__link_line_width') + """,
linkColor: '""" + get_conf('diagram__link_color') + """',
nodes: {nodes},
showCheckbox: true,
colorByPoint: false,
color: '#007ad0',
dataLabels: {
color: 'white',
},
borderColor: 'white',
nodeWidth: """ + get_conf('diagram__node_width') + """,
nodePadding: """ + get_conf('diagram__node_padding') + """
}],
tooltip: {
outside: true,
def get_image(extension):
path = get_conf(f'views__templates__extension_images__{extension}')
return path
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.template_name = get_conf('views__templates__object_delete')
self.success_url = reverse_lazy(self.success_url)
class TaskUpdateView(ProcessGenericUpdateView):
model = Task
template_name = get_conf('views__templates__task_edit')
success_url = get_conf('views__task__update__success_url')
success_message = get_conf('views__task__update__success_message')
permissions = get_conf('views__task__update__permissions')
fields = [
'name',
'description',
'is_active',
'level',
'offset',
'interpreter',
'arguments',
'code',
]
def get_context_data(self, **kwargs):
if 'form' not in kwargs:
kwargs['form'] = self.get_form()
if 'parents' not in kwargs:
kwargs['parents'] = self.object.parents.all().\
extra(select={"badge": "'primary'", 'is_new': "''"}).\
values('parent__id', 'parent__name', 'badge', 'is_new')
exclude_ids = [i['parent__id'] for i in kwargs['parents']]
exclude_ids += [self.object.id]
if 'new_parents' in kwargs:
exclude_ids += [i['parent__id'] for i in kwargs['new_parents']]
# we get the tasks which can be parent
kwargs['parent_options'] = self.object.process.tasks.all(). \
exclude(id__in=exclude_ids). \
values('id', 'name')
return super().get_context_data(**kwargs)
def post(self, request, *args, **kwargs):
self.object = self.get_object()
form = self.get_form()
parents = self.object.parents.filter(parent__id__in=request.POST.getlist('parents')).\
extra(select={"badge": "'primary'", 'is_new': "''"}).\
values('parent__id', 'parent__name', 'badge', 'is_new')
new_parents = Task.objects.filter(id__in=request.POST.getlist('new-parents')). \
extra(select={'parent__id': 'id', 'parent__name': 'name', 'badge': "'info'", 'is_new': "'new-'"}). \
values('parent__id', 'parent__name', 'badge', 'is_new')
response = self.render_to_response(
self.get_context_data(form=form,
parents=parents,
new_parents=new_parents))
if not form.is_valid():
return response
try:
with transaction.atomic():
self.object = form.save()
# get or create parent relation
for new_parent in new_parents:
new_parent['badge'] = 'danger'
parent_task = get_object_or_404(
Task, id=new_parent['parent__id'])
obj, created = TaskDependence.objects.get_or_create(
task=self.object, parent=parent_task)
new_parent['badge'] = 'primary'
for current in self.object.parents.all():
if not current.parent.id in [i['parent__id'] for i in parents] + \
[i['parent__id'] for i in new_parents]:
current.delete()
messages.success(request, self.success_message)
return redirect('process-tasks-update', pk=self.object.id)
except ValidationError as e:
for msg in e.messages:
messages.error(request, _(f'{msg}'))
return response
except Exception as e:
messages.error(request, _(f'{e}'))
return response
class ProcessDeleteView(ProcessGenericDeleteView):
model = Process
success_url = get_conf('views__process__delete__success_url')
success_message = get_conf('views__process__delete__success_message')
permissions = get_conf('views__process__delete__permissions')
class TaskListView(ProcessGenericListView):
model = Task
title = 'Tasks'
filters = get_conf('views__task__list__url_allow_filters')
permissions = get_conf('views__task__list__permissions')
