def test_related_object(self):
"""Test that the `related_object` property is computed properly."""
# Case 1: Job, identified by class_path.
with self.settings(JOBS_ROOT=os.path.join(settings.BASE_DIR,
"extras/tests/dummy_jobs")):
job_class = get_job("local/test_pass/TestPass")
job_result = JobResult(
name=job_class.class_path,
obj_type=ContentType.objects.get(app_label="extras",
model="job"),
job_id=uuid.uuid4(),
)
# Can't just do self.assertEqual(job_result.related_object, job_class) here for some reason
self.assertEqual(type(job_result.related_object), type)
self.assertTrue(issubclass(job_result.related_object, Job))
self.assertEqual(job_result.related_object.class_path,
"local/test_pass/TestPass")
job_result.name = "local/no_such_job/NoSuchJob"
self.assertIsNone(job_result.related_object)
job_result.name = "not-a-class-path"
self.assertIsNone(job_result.related_object)
# Case 2: GitRepository, identified by name.
repo = GitRepository(
name="Test Git Repository",
slug="test-git-repo",
remote_url="http://localhost/git.git",
username="******",
)
repo.save(trigger_resync=False)
job_result = JobResult(
name=repo.name,
obj_type=ContentType.objects.get_for_model(repo),
job_id=uuid.uuid4(),
)
self.assertEqual(job_result.related_object, repo)
job_result.name = "No such GitRepository"
self.assertIsNone(job_result.related_object)
# Case 3: Related object with no name, identified by PK/ID
ip_address = IPAddress.objects.create(address="1.1.1.1/32")
job_result = JobResult(
name="irrelevant",
obj_type=ContentType.objects.get_for_model(ip_address),
job_id=ip_address.pk,
)
self.assertEqual(job_result.related_object, ip_address)
job_result.job_id = uuid.uuid4()
self.assertIsNone(job_result.related_object)
def enqueue_pull_git_repository_and_refresh_data(repository, request):
"""
Convenience wrapper for JobResult.enqueue_job() to enqueue the pull_git_repository_and_refresh_data job.
"""
git_repository_content_type = ContentType.objects.get_for_model(GitRepository)
JobResult.enqueue_job(
pull_git_repository_and_refresh_data,
repository.name,
git_repository_content_type,
request.user,
repository_pk=repository.pk,
request=copy_safe_request(request),
)
def enqueue_git_repository_helper(repository, request, func, **kwargs):
"""
Wrapper for JobResult.enqueue_job() to enqueue one of several possible Git repository functions.
"""
git_repository_content_type = ContentType.objects.get_for_model(
GitRepository)
JobResult.enqueue_job(
func,
repository.name,
git_repository_content_type,
request.user,
repository_pk=repository.pk,
request=copy_safe_request(request),
)
def dry_run(self, request, pk):
if not request.user.has_perm("extras.run_job"):
raise PermissionDenied()
scheduled_job = get_object_or_404(ScheduledJob, pk=pk)
job_class = get_job(scheduled_job.job_class)
if job_class is None:
raise Http404
job = job_class()
grouping, module, class_name = job_class.class_path.split("/", 2)
# Immediately enqueue the job with commit=False
job_content_type = ContentType.objects.get(app_label="extras",
model="job")
job_result = JobResult.enqueue_job(
run_job,
job.class_path,
job_content_type,
scheduled_job.user,
data=scheduled_job.kwargs["data"],
request=copy_safe_request(request),
commit=False, # force a dry-run
)
serializer = serializers.JobResultSerializer(
job_result, context={"request": request})
return Response(serializer.data)
def run(self, request, class_path):
if not request.user.has_perm("extras.run_job"):
raise PermissionDenied("This user does not have permission to run jobs.")
job_class = self._get_job_class(class_path)
job = job_class()
input_serializer = serializers.JobInputSerializer(data=request.data)
input_serializer.is_valid(raise_exception=True)
data = input_serializer.data["data"]
commit = input_serializer.data["commit"]
if commit is None:
commit = getattr(job_class.Meta, "commit_default", True)
job_content_type = ContentType.objects.get(app_label="extras", model="job")
job_result = JobResult.enqueue_job(
run_job,
job.class_path,
job_content_type,
request.user,
data=data,
request=copy_safe_request(request),
commit=commit,
)
job.result = job_result
serializer = serializers.JobDetailSerializer(job, context={"request": request})
return Response(serializer.data)
def scheduled_job_handler(*args, **kwargs):
"""
A thin wrapper around JobResult.enqueue_job() that allows for it to be called as an async task
for the purposes of enqueuing scheduled jobs at their recurring intervals. Thus, JobResult.enqueue_job()
is responsible for enqueuing the actual job for execution and this method is the task executed
by the scheduler to kick off the job execution on a recurring interval.
"""
from nautobot.extras.models import JobResult # avoid circular import
user_pk = kwargs.pop("user")
user = User.objects.get(pk=user_pk)
name = kwargs.pop("name")
scheduled_job_pk = kwargs.pop("scheduled_job_pk")
schedule = ScheduledJob.objects.get(pk=scheduled_job_pk)
job_content_type = ContentType.objects.get(app_label="extras", model="job")
JobResult.enqueue_job(run_job, name, job_content_type, user, schedule=schedule, **kwargs)
def run(self, request, class_path):
if not request.user.has_perm("extras.run_job"):
raise PermissionDenied(
"This user does not have permission to run jobs.")
job_class = self._get_job_class(class_path)
job = job_class()
input_serializer = serializers.JobInputSerializer(data=request.data)
input_serializer.is_valid(raise_exception=True)
data = input_serializer.data["data"] or {}
commit = input_serializer.data["commit"]
if commit is None:
commit = getattr(job_class.Meta, "commit_default", True)
try:
job.validate_data(data)
except FormsValidationError as e:
# message_dict can only be accessed if ValidationError got a dict
# in the constructor (saved as error_dict). Otherwise we get a list
# of errors under messages
return Response(
{
"errors":
e.message_dict if hasattr(e, "error_dict") else e.messages
},
status=400)
if not get_worker_count():
raise CeleryWorkerNotRunningException()
job_content_type = ContentType.objects.get(app_label="extras",
model="job")
schedule = input_serializer.data.get("schedule")
if schedule:
schedule = self._create_schedule(schedule, data, commit, job,
job_class, request)
else:
job_result = JobResult.enqueue_job(
run_job,
job.class_path,
job_content_type,
request.user,
data=data,
request=copy_safe_request(request),
commit=commit,
)
job.result = job_result
serializer = serializers.JobDetailSerializer(
job, context={"request": request})
return Response(serializer.data)
def setUp(self):
self.user = User.objects.create_user(username="******")
self.factory = RequestFactory()
self.dummy_request = self.factory.get("/no-op/")
self.dummy_request.user = self.user
# Needed for use with the change_logging decorator
self.dummy_request.id = uuid.uuid4()
self.site = Site.objects.create(name="Test Site", slug="test-site")
self.manufacturer = Manufacturer.objects.create(name="Acme",
slug="acme")
self.device_type = DeviceType.objects.create(
manufacturer=self.manufacturer,
model="Frobozz 1000",
slug="frobozz1000")
self.role = DeviceRole.objects.create(name="router", slug="router")
self.device_status = Status.objects.get_for_model(Device).get(
slug="active")
self.device = Device.objects.create(
name="test-device",
device_role=self.role,
device_type=self.device_type,
site=self.site,
status=self.device_status,
)
self.repo = GitRepository(
name="Test Git Repository",
slug="test_git_repo",
remote_url="http://localhost/git.git",
# Provide everything we know we can provide
provided_contents=[
entry.content_identifier
for entry in get_datasource_contents("extras.gitrepository")
],
)
self.repo.save(trigger_resync=False)
self.job_result = JobResult(
name=self.repo.name,
obj_type=ContentType.objects.get_for_model(GitRepository),
job_id=uuid.uuid4(),
)
def run(self, request, class_path):
if not request.user.has_perm("extras.run_job"):
raise PermissionDenied(
"This user does not have permission to run jobs.")
# Check that at least one RQ worker is running
if not Worker.count(get_connection("default")):
raise RQWorkerNotRunningException()
job_class = self._get_job_class(class_path)
job = job_class()
input_serializer = serializers.JobInputSerializer(data=request.data)
if not input_serializer.is_valid():
return Response(input_serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
data = input_serializer.data["data"]
commit = input_serializer.data["commit"]
if commit is None:
commit = getattr(job_class.Meta, "commit_default", True)
job_content_type = ContentType.objects.get(app_label="extras",
model="job")
job_result = JobResult.enqueue_job(
run_job,
job.class_path,
job_content_type,
request.user,
data=data,
request=copy_safe_request(request),
commit=commit,
)
job.result = job_result
serializer = serializers.JobDetailSerializer(
job, context={"request": request})
return Response(serializer.data)
def dry_run(self, request, pk):
scheduled_job = get_object_or_404(ScheduledJob, pk=pk)
job_model = scheduled_job.job_model
if job_model is None or not job_model.runnable:
raise MethodNotAllowed("This job cannot be dry-run at this time.")
if not Job.objects.check_perms(request.user, instance=job_model, action="run"):
raise PermissionDenied("You do not have permission to run this job.")
# Immediately enqueue the job with commit=False
job_content_type = get_job_content_type()
job_result = JobResult.enqueue_job(
run_job,
job_model.class_path,
job_content_type,
request.user,
data=scheduled_job.kwargs.get("data", {}),
request=copy_safe_request(request),
commit=False, # force a dry-run
)
serializer = serializers.JobResultSerializer(job_result, context={"request": request})
return Response(serializer.data)
def handle(self, *args, **options):
if "/" not in options["job"]:
raise CommandError(
'Job must be specified in the form "grouping_name/module_name/JobClassName"'
)
job_class = get_job(options["job"])
if not job_class:
raise CommandError('Job "%s" not found' % options["job"])
job_content_type = ContentType.objects.get(app_label="extras",
model="job")
# Run the job and create a new JobResult
self.stdout.write("[{:%H:%M:%S}] Running {}...".format(
timezone.now(), job_class.class_path))
job_result = JobResult.enqueue_job(
run_job,
job_class.class_path,
job_content_type,
None,
data=
{}, # TODO: parsing CLI args into a data dictionary is not currently implemented
request=None,
commit=options["commit"],
)
# Wait on the job to finish
while job_result.status not in JobResultStatusChoices.TERMINAL_STATE_CHOICES:
time.sleep(1)
job_result = JobResult.objects.get(pk=job_result.pk)
# Report on success/failure
for test_name, attrs in job_result.data.items():
if test_name in ["total", "output"]:
continue
self.stdout.write(
"\t{}: {} success, {} info, {} warning, {} failure".format(
test_name,
attrs["success"],
attrs["info"],
attrs["warning"],
attrs["failure"],
))
for log_entry in attrs["log"]:
status = log_entry[1]
if status == "success":
status = self.style.SUCCESS(status)
elif status == "info":
status = status
elif status == "warning":
status = self.style.WARNING(status)
elif status == "failure":
status = self.style.NOTICE(status)
if log_entry[2]: # object associated with log entry
self.stdout.write(
f"\t\t{status}: {log_entry[2]}: {log_entry[-1]}")
else:
self.stdout.write(f"\t\t{status}: {log_entry[-1]}")
if job_result.data["output"]:
self.stdout.write(job_result.data["output"])
if job_result.status == JobResultStatusChoices.STATUS_FAILED:
status = self.style.ERROR("FAILED")
elif job_result.status == JobResultStatusChoices.STATUS_ERRORED:
status = self.style.ERROR("ERRORED")
else:
status = self.style.SUCCESS("SUCCESS")
self.stdout.write("[{:%H:%M:%S}] {}: {}".format(
timezone.now(), job_class.class_path, status))
# Wrap things up
self.stdout.write("[{:%H:%M:%S}] {}: Duration {}".format(
timezone.now(), job_class.class_path, job_result.duration))
self.stdout.write("[{:%H:%M:%S}] Finished".format(timezone.now()))
def _run_job(request, job_model, legacy_response=False):
"""An internal function providing logic shared between JobModelViewSet.run() and JobViewSet.run()."""
if not request.user.has_perm("extras.run_job"):
raise PermissionDenied("This user does not have permission to run jobs.")
if not job_model.enabled:
raise PermissionDenied("This job is not enabled to be run.")
if not job_model.installed:
raise MethodNotAllowed(request.method, detail="This job is not presently installed and cannot be run")
job_class = job_model.job_class
if job_class is None:
raise MethodNotAllowed(request.method, detail="This job's source code could not be located and cannot be run")
job = job_class()
input_serializer = serializers.JobInputSerializer(data=request.data)
input_serializer.is_valid(raise_exception=True)
data = input_serializer.data["data"] or {}
commit = input_serializer.data["commit"]
if commit is None:
commit = job_model.commit_default
try:
job.validate_data(data)
except FormsValidationError as e:
# message_dict can only be accessed if ValidationError got a dict
# in the constructor (saved as error_dict). Otherwise we get a list
# of errors under messages
return Response({"errors": e.message_dict if hasattr(e, "error_dict") else e.messages}, status=400)
if not get_worker_count():
raise CeleryWorkerNotRunningException()
job_content_type = get_job_content_type()
schedule_data = input_serializer.data.get("schedule")
# Default to a null JobResult.
job_result = None
# Assert that a job with `approval_required=True` has a schedule that enforces approval and
# executes immediately.
if schedule_data is None and job_model.approval_required:
schedule_data = {"interval": JobExecutionType.TYPE_IMMEDIATELY}
# Try to create a ScheduledJob, or...
if schedule_data:
schedule = _create_schedule(schedule_data, data, commit, job, job_model, request)
else:
schedule = None
# ... If we can't create one, create a JobResult instead.
if schedule is None:
job_result = JobResult.enqueue_job(
run_job,
job.class_path,
job_content_type,
request.user,
data=data,
request=copy_safe_request(request),
commit=commit,
)
job.result = job_result
if legacy_response:
# Old-style JobViewSet response - serialize the Job class in the response for some reason?
serializer = serializers.JobClassDetailSerializer(job, context={"request": request})
return Response(serializer.data)
else:
# New-style JobModelViewSet response - serialize the schedule or job_result as appropriate
data = {"schedule": None, "job_result": None}
if schedule:
data["schedule"] = nested_serializers.NestedScheduledJobSerializer(
schedule, context={"request": request}
).data
if job_result:
data["job_result"] = nested_serializers.NestedJobResultSerializer(
job_result, context={"request": request}
).data
return Response(data, status=status.HTTP_201_CREATED)
def test_pull_git_repository_and_refresh_data_with_valid_data(
self, MockGitRepo):
"""
The test_pull_git_repository_and_refresh_data job should succeed if valid data is present in the repo.
"""
with tempfile.TemporaryDirectory() as tempdir:
with self.settings(GIT_ROOT=tempdir):
def populate_repo(path, url):
os.makedirs(path)
# Just make config_contexts and export_templates directories as we don't load jobs
os.makedirs(os.path.join(path, "config_contexts"))
os.makedirs(
os.path.join(path, "config_contexts", "devices"))
os.makedirs(
os.path.join(path, "export_templates", "dcim",
"device"))
with open(
os.path.join(path, "config_contexts",
"context.yaml"), "w") as fd:
yaml.dump(
{
"_metadata": {
"name": "Region NYC servers",
"weight": 1500,
"description":
"NTP servers for region NYC",
"is_active": True,
},
"ntp-servers":
["172.16.10.22", "172.16.10.33"],
},
fd,
)
with open(
os.path.join(path, "config_contexts", "devices",
"test-device.json"),
"w",
) as fd:
json.dump({"dns-servers": ["8.8.8.8"]}, fd)
with open(
os.path.join(path, "export_templates", "dcim",
"device", "template.j2"),
"w",
) as fd:
fd.write(
"{% for device in queryset %}\n{{ device.name }}\n{% endfor %}"
)
return mock.DEFAULT
MockGitRepo.side_effect = populate_repo
MockGitRepo.return_value.checkout.return_value = self.COMMIT_HEXSHA
pull_git_repository_and_refresh_data(self.repo.pk,
self.dummy_request,
self.job_result)
self.assertEqual(
self.job_result.status,
JobResultStatusChoices.STATUS_COMPLETED,
self.job_result.data,
)
# Make sure ConfigContext was successfully loaded from file
config_context = ConfigContext.objects.get(
name="Region NYC servers",
owner_object_id=self.repo.pk,
owner_content_type=ContentType.objects.get_for_model(
GitRepository),
)
self.assertIsNotNone(config_context)
self.assertEqual(1500, config_context.weight)
self.assertEqual("NTP servers for region NYC",
config_context.description)
self.assertTrue(config_context.is_active)
self.assertEqual(
{"ntp-servers": ["172.16.10.22", "172.16.10.33"]},
config_context.data,
)
# Make sure Device local config context was successfully populated from file
device = Device.objects.get(name=self.device.name)
self.assertIsNotNone(device.local_context_data)
self.assertEqual({"dns-servers": ["8.8.8.8"]},
device.local_context_data)
self.assertEqual(device.local_context_data_owner, self.repo)
# Make sure ExportTemplate was successfully loaded from file
export_template = ExportTemplate.objects.get(
owner_object_id=self.repo.pk,
owner_content_type=ContentType.objects.get_for_model(
GitRepository),
content_type=ContentType.objects.get_for_model(Device),
name="template.j2",
)
self.assertIsNotNone(export_template)
# Now "resync" the repository, but now those files no longer exist in the repository
def empty_repo(path, url):
os.remove(
os.path.join(path, "config_contexts", "context.yaml"))
os.remove(
os.path.join(path, "config_contexts", "devices",
"test-device.json"))
os.remove(
os.path.join(path, "export_templates", "dcim",
"device", "template.j2"))
return mock.DEFAULT
MockGitRepo.side_effect = empty_repo
# For verisimilitude, don't re-use the old request and job_result
self.dummy_request.id = uuid.uuid4()
self.job_result = JobResult(
name=self.repo.name,
obj_type=ContentType.objects.get_for_model(GitRepository),
job_id=uuid.uuid4(),
)
pull_git_repository_and_refresh_data(self.repo.pk,
self.dummy_request,
self.job_result)
self.assertEqual(
self.job_result.status,
JobResultStatusChoices.STATUS_COMPLETED,
self.job_result.data,
)
# Verify that objects have been removed from the database
self.assertEqual(
[],
list(
ConfigContext.objects.filter(
owner_content_type=ContentType.objects.
get_for_model(GitRepository),
owner_object_id=self.repo.pk,
)),
)
self.assertEqual(
[],
list(
ExportTemplate.objects.filter(
owner_content_type=ContentType.objects.
get_for_model(GitRepository),
owner_object_id=self.repo.pk,
)),
)
device = Device.objects.get(name=self.device.name)
self.assertIsNone(device.local_context_data)
self.assertIsNone(device.local_context_data_owner)
def job_runner(handle_class, job_class, device=None, user=None):
"""Function to make management command code more DRY."""
data = {}
if device:
data["device"] = Device.objects.get(name=device)
request = RequestFactory().request(SERVER_NAME="WebRequestContext")
request.id = uuid.uuid4()
request.user = User.objects.get(username=user)
job_content_type = ContentType.objects.get(app_label="extras", model="job")
# Run the job and create a new JobResult
handle_class.stdout.write("[{:%H:%M:%S}] Running {}...".format(
timezone.now(), job_class.class_path))
job_result = JobResult.enqueue_job(
run_job,
job_class.class_path,
job_content_type,
request.user,
data=data,
request=request,
commit=True,
)
# Wait on the job to finish
while job_result.status not in JobResultStatusChoices.TERMINAL_STATE_CHOICES:
time.sleep(1)
job_result = JobResult.objects.get(pk=job_result.pk)
# Report on success/failure
for test_name, attrs in job_result.data.items():
if test_name in ["total", "output"]:
continue
handle_class.stdout.write(
"\t{}: {} success, {} info, {} warning, {} failure".format(
test_name,
attrs["success"],
attrs["info"],
attrs["warning"],
attrs["failure"],
))
for log_entry in attrs["log"]:
status = log_entry[1]
if status == "success":
status = handle_class.style.SUCCESS(status)
elif status == "info":
status = status # pylint: disable=self-assigning-variable
elif status == "warning":
status = handle_class.style.WARNING(status)
elif status == "failure":
status = handle_class.style.NOTICE(status)
if log_entry[2]: # object associated with log entry
handle_class.stdout.write(
f"\t\t{status}: {log_entry[2]}: {log_entry[-1]}")
else:
handle_class.stdout.write(f"\t\t{status}: {log_entry[-1]}")
if job_result.data["output"]:
handle_class.stdout.write(job_result.data["output"])
if job_result.status == JobResultStatusChoices.STATUS_FAILED:
status = handle_class.style.ERROR("FAILED")
elif job_result.status == JobResultStatusChoices.STATUS_ERRORED:
status = handle_class.style.ERROR("ERRORED")
else:
status = handle_class.style.SUCCESS("SUCCESS")
handle_class.stdout.write("[{:%H:%M:%S}] {}: {}".format(
timezone.now(), job_class.class_path, status))
# Wrap things up
handle_class.stdout.write("[{:%H:%M:%S}] {}: Duration {}".format(
timezone.now(), job_class.class_path, job_result.duration))
handle_class.stdout.write("[{:%H:%M:%S}] Finished".format(timezone.now()))
def handle(self, *args, **options):
if "/" not in options["job"]:
raise CommandError(
'Job must be specified in the form "grouping_name/module_name/JobClassName"'
)
job_class = get_job(options["job"])
if not job_class:
raise CommandError('Job "%s" not found' % options["job"])
user = None
request = None
if options["commit"] and not options["username"]:
# Job execution with commit=True uses change_logging(), which requires a user as the author of any changes
raise CommandError("--username is mandatory when --commit is used")
if options["username"]:
User = get_user_model()
try:
user = User.objects.get(username=options["username"])
except User.DoesNotExist as exc:
raise CommandError("No such user") from exc
request = RequestFactory().request(
SERVER_NAME="nautobot_server_runjob")
request.id = uuid.uuid4()
request.user = user
job_content_type = get_job_content_type()
# Run the job and create a new JobResult
self.stdout.write("[{:%H:%M:%S}] Running {}...".format(
timezone.now(), job_class.class_path))
job_result = JobResult.enqueue_job(
run_job,
job_class.class_path,
job_content_type,
user,
data=
{}, # TODO: parsing CLI args into a data dictionary is not currently implemented
request=copy_safe_request(request) if request else None,
commit=options["commit"],
)
# Wait on the job to finish
while job_result.status not in JobResultStatusChoices.TERMINAL_STATE_CHOICES:
time.sleep(1)
job_result = JobResult.objects.get(pk=job_result.pk)
# Report on success/failure
groups = set(
JobLogEntry.objects.filter(job_result=job_result).values_list(
"grouping", flat=True))
for group in sorted(groups):
logs = JobLogEntry.objects.filter(job_result__pk=job_result.pk,
grouping=group)
success_count = logs.filter(
log_level=LogLevelChoices.LOG_SUCCESS).count()
info_count = logs.filter(
log_level=LogLevelChoices.LOG_INFO).count()
warning_count = logs.filter(
log_level=LogLevelChoices.LOG_WARNING).count()
failure_count = logs.filter(
log_level=LogLevelChoices.LOG_FAILURE).count()
self.stdout.write(
"\t{}: {} success, {} info, {} warning, {} failure".format(
group,
success_count,
info_count,
warning_count,
failure_count,
))
for log_entry in logs:
status = log_entry.log_level
if status == "success":
status = self.style.SUCCESS(status)
elif status == "info":
status = status
elif status == "warning":
status = self.style.WARNING(status)
elif status == "failure":
status = self.style.NOTICE(status)
if log_entry.log_object:
self.stdout.write(
f"\t\t{status}: {log_entry.log_object}: {log_entry.message}"
)
else:
self.stdout.write(f"\t\t{status}: {log_entry.message}")
if job_result.data["output"]:
self.stdout.write(job_result.data["output"])
if job_result.status == JobResultStatusChoices.STATUS_FAILED:
status = self.style.ERROR("FAILED")
elif job_result.status == JobResultStatusChoices.STATUS_ERRORED:
status = self.style.ERROR("ERRORED")
else:
status = self.style.SUCCESS("SUCCESS")
self.stdout.write("[{:%H:%M:%S}] {}: {}".format(
timezone.now(), job_class.class_path, status))
# Wrap things up
self.stdout.write("[{:%H:%M:%S}] {}: Duration {}".format(
timezone.now(), job_class.class_path, job_result.duration))
self.stdout.write("[{:%H:%M:%S}] Finished".format(timezone.now()))